#11 New compose ID parser, handle Bikeshed modular nightlies, explicitly don't support updates composes
Closed 6 years ago by adamwill. Opened 6 years ago by adamwill.

file modified
+115 -19
@@ -409,7 +409,7 @@ 

      else:

          return ''

  

- def parse_cid(origcid, dist=False):

+ def parse_cid(origcid, dist=False, dic=False):

      """If dist is false-y, get (release, date, type, respin) values

      from a Pungi 4 compose id. If dist is truth-y, get (dist, release,

      date, type, respin) values from a Pungi 4 compose id.
@@ -419,26 +419,122 @@ 

      if not cid:

          raise ValueError("{0} does not appear to be a valid Pungi 4 Fedora "

                           "compose ID!".format(origcid))

-     # The dist can have '-' in it, of course, because life is awful

-     # FIXME: This is very stupid and will break on layered products,

-     # which have a lot more - in them. It can theoretically break in

-     # several other cases, but I don't know if Fedora is currently

-     # hitting any of those. I'm currently thinking about how to deal

-     # with this. See:

+     # Writing a truly generic compose ID parser is pretty tricky. See

      # https://github.com/release-engineering/productmd/issues/77

-     # We may just have to rejig things to require the full compose

-     # metadata and give up on parsing compose IDs at all. That would

-     # be a lot of work for fedfind. A somewhat uglier alternative for

-     # fedfind's purposes would be to give it special knowledge of the

-     # Fedora product shortnames with - in them, or a generic handler

-     # for 'Fedora-(FOO)' or something.

-     (gotdist, release, _) = cid.rsplit('-', 2)

-     release = release.lower()

+     # for some background. This is my best effort. I'm trying to get

+     # it upstreamed currently.

+ 

+     # init values

+     mainshort = ''

+     mainver = ''

+     maintype = ''

+     baseshort = ''

+     basever = ''

+     basetype = ''

+     variant = ''

+ 

+     # find date, type, respin

      (date, typ, respin) = productmd.composeinfo.get_date_type_respin(cid)

-     if dist:

-         return (gotdist, release, date, typ, respin)

+     # now split on the date, we only care about what comes before

+     part = cid.split(date)[0][:-1]

+ 

+     # Handle "HACK: there are 2 RHEL 5 composes"

+     if part.endswith("-Client"):

+         variant = "Client"

+         part = part[:-len('-Client')]

+     elif part.endswith("-Server"):

+         variant = "Server"

+         part = part[:-len('-Server')]

+ 

+     # Next part back must be either a version type suffix or a version

+     # we don't know yet if this is the main version or the base

+     # version for a layered product

+     sometype = ''

+     # copy the list of release types and sort it in descending order

+     # by length, so we won't match on a subset of the entire type.

+     _reltypes = sorted(productmd.common.RELEASE_TYPES, key=lambda x: len(x), reverse=True)

+     for _reltype in _reltypes:

+         # if part ends in _reltype, take it as 'sometype' and cut it

+         if part.lower().endswith(_reltype.lower()):

+             sometype = _reltype

+             part = part[:-len(_reltype)-1]

+             break

+     # now whatever's at the end of 'part' must be a version

+     somever = part.split('-')[-1]

+     # strip it off

+     part = '-'.join(part.split('-')[:-1])

+ 

+     # what remains is either:

+     # mainshort

+     # or:

+     # mainshort-mainver(-maintype)-baseshort

+     # But this is where things get fun, because sadly, both mainshort

+     # and baseshort could have - in them and mainver could have a type

+     # suffix. So, life is fun. Working in our favour, shortnames are

+     # *not* allowed to contain digits. Let's see if we can spot what

+     # looks like a '-mainver'- component...

+     elems = part.split('-')

+     for (idx, cand) in enumerate(elems):

+         # can't be the first or the last

+         if idx == 0 or idx == len(elems) - 1:

+             continue

+         # now see if the cand looks like a version

+         # FIXME: we should use RELEASE_VERSION_RE here, but not until

+         # it enforces this stricter rule. We have decided that only

+         # 'Rawhide' or anything that looks like a version number -

+         # starts with a digit, can have any number of subsequent

+         # digits and/or digit groups separated with single periods -

+         # is a valid version.

+         verreg = re.compile(r'(Rawhide|Bikeshed|[0-9]+(\.?[0-9]+)*)$')

+         match = verreg.match(cand)

+         if match:

+             mainver = match.group(1)

+             # check for a type suffix. of course, because life is

+             # awful, some of these have dashes in them, so we get to

+             # be careful...

+             for _reltype in _reltypes:

+                 rtelems = _reltype.lower().split('-')

+                 if elems[idx+1:idx+1+len(rtelems)] == rtelems:

+                     maintype = _reltype

+                     # pop out the right number of elems

+                     for i in range(0,len(rtelems)):

+                         elems.pop(idx+1)

+                     break

+             basever = somever

+             basetype = sometype

+             mainshort = '-'.join(elems[:idx])

+             baseshort = '-'.join(elems[idx+1:])

+             break

+ 

+     # if we didn't establish a mainver above, we must not be layered,

+     # and what remains is just mainshort, and somever is mainver

+     if not mainshort:

+         mainshort = '-'.join(elems)

+         mainver = somever

+         maintype = sometype

+ 

+     if not maintype:

+         maintype = 'ga'

+     if basever and not basetype:

+         basetype = 'ga'

+ 

+     if dic:

+         return {

+             'short': mainshort,

+             'version': mainver,

+             'version_type': maintype,

+             'base_short': baseshort,

+             'base_version': basever,

+             'base_type': basetype,

+             'variant': variant,

+             'date': date,

+             'compose_type': typ,

+             'respin': respin

+         }

+     elif dist:

+         return (mainshort, mainver.lower(), date, typ, respin)

      else:

-         return (release, date, typ, respin)

+         return (mainver.lower(), date, typ, respin)

  

  def cid_from_label(release, label, short='fedora'):

      """Get the compose ID for a compose by label. Must also specify
@@ -462,7 +558,7 @@ 

      production composes have discoverable labels; will return the

      empty string for other composes.

      """

-     typ = parse_cid(cid)[2]

+     typ = parse_cid(cid, dic=True)['compose_type']

      if typ != 'production':

          return ''

      params = {'compose_id': cid}

file modified
+57 -10
@@ -192,7 +192,7 @@ 

          otherwise we leave it blank (this is so e.g. (22, '', '')

          returns Fedora 22).

          """

-         snaps = ('rawhide', 'branched', 'atomic', 'docker', 'cloud', 'production')

+         snaps = ('rawhide', 'bikeshed', 'branched', 'atomic', 'docker', 'cloud', 'production')

          if not compose and not_or_str((release, milestone), snaps):

              logger.debug("Guessing date")

              compose = fedfind.helpers.date_check(
@@ -214,11 +214,11 @@ 

          # it.

          if release:

              check = str(release).lower()

-             if check.isdigit() or check == 'rawhide':

+             if check.isdigit() or check in ('rawhide', 'bikeshed'):

                  return release

              else:

                  raise ValueError("get_release(): Release must be a number "

-                                  "or Rawhide!")

+                                  "or Rawhide or Bikeshed!")

  

          # If we were specifically asked for a valid milestone, return

          # appropriate release.
@@ -279,6 +279,8 @@ 

                 testcls = partial(RawhideModularNightly, compose=date)

              else:

                  testcls = partial(RawhideNightly, compose=date)

+         elif release == 'Bikeshed' and dist.lower() == 'fedora-modular':

+             testcls = partial(BikeshedModularNightly, compose=date)

          elif milestone.lower() == 'atomic':

              testcls = partial(AtomicNightly, release=release, compose=date)

          elif milestone.lower() == 'docker':
@@ -368,7 +370,14 @@ 

          # produced the fake CID in the first place).

          """

          # Assume a real Pungi 4 compose ID

-         (dist, release, date, typ, respin) = fedfind.helpers.parse_cid(cid, dist=True)

+         ciddic = fedfind.helpers.parse_cid(cid, dic=True)

+         (dist, release, date, typ, respin) = (ciddic['short'], ciddic['version'], ciddic['date'],

+                                               ciddic['compose_type'], ciddic['respin'])

+         release = release.capitalize()

+ 

+         if ciddic['version_type'] in ('updates', 'updates-testing') and dist != "FedoraRespin":

+             raise ValueError("get_release(): fedfind does not support updates or updates-testing "

+                              "composes like {0} as they contain no images".format(cid))

  

          # Atomic stable nightly composes, 'Fedora-Atomic' shortname

          if dist == 'Fedora-Atomic':
@@ -387,9 +396,8 @@ 

              return (dist, release, 'Respin', date, 0)

  

          if typ == 'nightly':

-             if release == 'rawhide':

+             if release == 'Rawhide':

                  milestone = ''

-                 release = 'Rawhide'

              else:

                  milestone = 'Branched'

              return (dist, release, milestone, date, int(respin))
@@ -487,6 +495,14 @@ 

          else:

              return guess_nightly_respin(dist, release, milestone, compose, cid=cid)

  

+     # 'Bikeshed' modular nightly case (this is basically Rawhide, but

+     # releng decided to change the name for some goddamn reason).

+     if release == 'Bikeshed' and dist.lower() == 'fedora-modular':

+         if respin is not None:

+             return BikeshedModularNightly(compose, respin, cid=cid)

+         else:

+             return guess_nightly_respin(dist, release, milestone, compose, cid=cid)

+ 

      # All nightly cases. 'Production' also handled here, as 'compose'

      # is a date.

      if fedfind.helpers.date_check(compose, fail_raise=False):
@@ -948,7 +964,7 @@ 

          """

          if self._release:

              return self._release

-         return fedfind.helpers.parse_cid(self.cid)[0]

+         return fedfind.helpers.parse_cid(self.cid, dic=True)['version'].lower()

  

      @property

      def milestone(self):
@@ -1026,7 +1042,7 @@ 

          etc. In productmd terms this is in fact the 'short name', but

          Fedora releng decided not to make them very short...

          """

-         return fedfind.helpers.parse_cid(self.cid, dist=True)[0]

+         return fedfind.helpers.parse_cid(self.cid, dic=True)['short']

  

      @property

      def previous_release(self):
@@ -1106,7 +1122,12 @@ 

  

  

  class RawhideModularNightly(Pungi4Release):

-     """Rawhide modular "nightly" (bit of a misnomer, now) composes."""

+     """Rawhide modular "nightly" (bit of a misnomer, now) composes.

+     Note these were replaced by BikeshedModularNightly between

+     20171003 and 20171013, there are none of these composes after

+     that date; once the earlier composes are removed this class will

+     likely be turned into a redirect to BikeshedModularNightly.

+     """

      def __init__(self, compose, respin=0, cid=None):

          compose = str(compose)

          respin = str(respin)
@@ -1128,6 +1149,32 @@ 

          )

  

  

+ class BikeshedModularNightly(Pungi4Release):

+     """Bikeshed modular "nightly" (bit of a misnomer, now) composes.

+     This is basically Rawhide for modular, I don't know why releng

+     decided to change the name other than to screw with me...

+     """

+     def __init__(self, compose, respin=0, cid=None):

+         compose = str(compose)

+         respin = str(respin)

+         path = "Fedora-Modular-Bikeshed-{0}.n.{1}/compose".format(

+             compose, respin)

+         url = '/'.join((fedfind.const.NIGHTLY_BASE, path))

+         super(BikeshedModularNightly, self).__init__(

+             url, release="Bikeshed", milestone='', compose=compose,

+             respin=respin, typ="nightly", cid=cid)

+ 

+     @property

+     def expected_images(self):

+         return (

+             ('docker_base', 'docker', 'x86_64'),

+             ('server', 'dvd', 'arm'),

+             ('server', 'dvd', 'x86_64'),

+             ('server', 'qcow2', 'x86_64'),

+             ('server', 'raw-xz', 'x86_64'),

+         )

+ 

+ 

  class BranchedNightly(Pungi4Release):

      """Branched "nightly" (bit of a misnomer, now) composes."""

      def __init__(self, release, compose, respin=0, cid=None):
@@ -1902,7 +1949,7 @@ 

          # if we got this far, we already know we exist

          self._exists = True

          # custom fake CID for these releases

-         self._fakecid = "FedoraRespin-{0}-{1}.0".format(self.release, self.compose)

+         self._fakecid = "FedoraRespin-{0}-updates-{1}.0".format(self.release, self.compose)

  

      def __repr__(self):

          return "{0}()".format(self.__class__)

file modified
+85 -16
@@ -301,6 +301,8 @@ 

          assert res == 'Fedora-Atomic-24-20161004.1'

          res = fedfind.helpers.find_cid('FedoraRespin-24-20161206.0')

          assert res == 'FedoraRespin-24-20161206.0'

+         res = fedfind.helpers.find_cid('FedoraRespin-24-updates-20161206.0')

+         assert res == 'FedoraRespin-24-updates-20161206.0'

          # a few 'close but no cigars' just for safety...

          res = fedfind.helpers.find_cid('https://foo.com/compose/Fedora-24-20160314.t.u/compose')

          assert res == ''
@@ -309,33 +311,100 @@ 

          res = fedfind.helpers.find_cid('https://foo.com/compose/Fedora/24-20160314.n.0/compose')

          assert res == ''

  

-     def test_parse_cid(self):

+     @pytest.mark.parametrize(

+         ("cid", "short", "version", "version_type", "base_short", "base_version", "base_type",

+          "variant", "date", "compose_type", "respin"),

+         [

+             ('Fedora-24-20160314.n.0', 'Fedora', '24', 'ga', '', '', '', '', '20160314', 'nightly',

+              0),

+             ('Fedora-24-20160314.0', 'Fedora', '24', 'ga', '', '', '', '', '20160314',

+              'production', 0),

+             ('Fedora-24-20160314.t.0', 'Fedora', '24', 'ga', '', '', '', '', '20160314', 'test', 0),

+             ('Fedora-Rawhide-20160314.n.1', 'Fedora', 'Rawhide', 'ga', '', '', '', '', '20160314',

+              'nightly', 1),

+             ('Fedora-Atomic-24-20160628.0', 'Fedora-Atomic', '24', 'ga', '', '', '', '',

+              '20160628', 'production', 0),

+             ('Fedora-20-19700101.0', 'Fedora', '20', 'ga', '', '', '', '', '19700101',

+              'production', 0),

+             ('FedoraRespin-24-updates-20161206.0', 'FedoraRespin', '24', 'updates', '', '', '', '',

+              '20161206', 'production', 0),

+             # older fake Respin CID

+             ('FedoraRespin-24-20161206.0', 'FedoraRespin', '24', 'ga', '', '', '', '', '20161206',

+              'production', 0),

+             # taken from productmd test_composeinfo

+             ('Fedora-22-20160622.0', 'Fedora', '22', 'ga', '', '', '', '', '20160622', 'production',

+              0),

+             # FIXME: 'ci' is a valid compose type now but get_date_type_respin doesn't handle it

+             # 'Fedora-22-20160622.ci.0'

+             ('Fedora-22-updates-20160622.0', 'Fedora', '22', 'updates', '', '', '', '', '20160622',

+              'production', 0),

+             ('Fedora-22-updates-20160622.n.0', 'Fedora', '22', 'updates', '', '', '', '',

+              '20160622', 'nightly', 0),

+             ('Fedora-22-BASE-3-20160622.0', 'Fedora', '22', 'ga', 'BASE', '3', 'ga', '', '20160622',

+              'production', 0),

+             ('Fedora-22-BASE-3-20160622.n.0', 'Fedora', '22', 'ga', 'BASE', '3', 'ga', '',

+              '20160622', 'nightly', 0),

+             ('Fedora-22-updates-BASE-3-20160622.0', 'Fedora', '22', 'updates', 'BASE', '3', 'ga',

+              '', '20160622', 'production', 0),

+             ('Fedora-22-updates-BASE-3-20160622.n.0', 'Fedora', '22', 'updates', 'BASE', '3', 'ga',

+              '', '20160622', 'nightly', 0),

+             ('Fedora-22-BASE-3-updates-20160622.0', 'Fedora', '22', 'ga', 'BASE', '3', 'updates',

+              '', '20160622', 'production', 0),

+             ('Fedora-22-BASE-3-updates-20160622.n.0', 'Fedora', '22', 'ga', 'BASE', '3', 'updates',

+              '', '20160622', 'nightly', 0),

+             ('Fedora-22-updates-BASE-3-updates-20160622.0', 'Fedora', '22', 'updates', 'BASE', '3',

+              'updates', '', '20160622', 'production', 0),

+             ('Fedora-22-updates-BASE-3-updates-20160622.n.0', 'Fedora', '22', 'updates', 'BASE',

+              '3', 'updates', '', '20160622', 'nightly', 0),

+             ('Fedora-22-updates-BASE-3-updates-testing-20160622.n.0', 'Fedora', '22',

+              'updates', 'BASE', '3', 'updates-testing', '', '20160622', 'nightly', 0),

+             ('Fedora-22-updates-testing-BASE-3-updates-20160622.n.0', 'Fedora', '22',

+              'updates-testing', 'BASE', '3', 'updates', '', '20160622', 'nightly', 0),

+             ('Fedora-22-updates-testing-BASE-3-updates-testing-20160622.n.0', 'Fedora', '22',

+              'updates-testing', 'BASE', '3', 'updates-testing', '', '20160622', 'nightly', 0),

+             # Rawhide complex

+             ('Fedora-Rawhide-updates-RHEL-6.3.4-20160513.t.1', 'Fedora', 'Rawhide', 'updates',

+              'RHEL', '6.3.4', 'ga', '', '20160513', 'test', 1),

+             # I think this is how a RHEL 5 one looks?

+             ('Fedora-5-updates-Server-20160523.2', 'Fedora', '5', 'updates', '', '', '', 'Server',

+              '20160523', 'production', 2),

+             ('Fedora-5-updates-Client-20160523.2', 'Fedora', '5', 'updates', '', '', '', 'Client',

+              '20160523', 'production', 2),

+         ]

+     )

+     def test_parse_cid(self, cid, short, version, version_type, base_short, base_version,

+                        base_type, variant, date, compose_type, respin):

          """Check parse_cid works correctly, with various known compose

          ID forms.

          """

+         assert fedfind.helpers.parse_cid(cid, dic=True) == {

+             'short': short,

+             'version': version,

+             'version_type': version_type,

+             'base_short': base_short,

+             'base_version': base_version,

+             'base_type': base_type,

+             'variant': variant,

+             'date': date,

+             'compose_type': compose_type,

+             'respin': respin

+         }

+ 

+     def test_parse_cid_tuple(self):

+         """Check the two older tuple formats for parse_cid work as

+         expected.

+         """

          res = fedfind.helpers.parse_cid('Fedora-24-20160314.n.0')

          assert res == ('24', '20160314', 'nightly', 0)

          res = fedfind.helpers.parse_cid('Fedora-24-20160314.n.0', dist=True)

          assert res == ('Fedora', '24', '20160314', 'nightly', 0)

-         res = fedfind.helpers.parse_cid('Fedora-24-20160314.0')

-         assert res == ('24', '20160314', 'production', 0)

-         res = fedfind.helpers.parse_cid('Fedora-24-20160314.t.0')

-         assert res == ('24', '20160314', 'test', 0)

-         res = fedfind.helpers.parse_cid('Fedora-Rawhide-20160314.n.1')

-         assert res == ('rawhide', '20160314', 'nightly', 1)

-         # Two-week Atomic uses a different product name

-         res = fedfind.helpers.parse_cid('Fedora-Atomic-24-20160628.0')

-         assert res == ('24', '20160628', 'production', 0)

          res = fedfind.helpers.parse_cid('Fedora-Atomic-24-20160628.0', dist=True)

          assert res == ('Fedora-Atomic', '24', '20160628', 'production', 0)

-         # should raise ValueError for non-Pungi 4-ish CID

+ 

+     def test_parse_cid_error(self):

+         """Check parse_cid raises ValueError for non-Pungi 4-ish CID."""

          with pytest.raises(ValueError):

              res = fedfind.helpers.parse_cid('23-20160530')

-         # should handle our own fake compose IDs, though

-         res = fedfind.helpers.parse_cid('Fedora-20-19700101.0', dist=True)

-         assert res == ('Fedora', '20', '19700101', 'production', 0)

-         res = fedfind.helpers.parse_cid('FedoraRespin-24-20161206.0', dist=True)

-         assert res == ('FedoraRespin', '24', '20161206', 'production', 0)

  

      @mock.patch('fedfind.helpers.download_json', return_value=PDC_JSON_REAL, autospec=True)

      def test_cid_from_label(self, fakejson):

file modified
+24 -3
@@ -282,6 +282,8 @@ 

              ('Rawhide', '', '', 0, "Fedora", fedfind.release.RawhideNightly, 'Rawhide', '', 'datestr', '0'),

              ('Rawhide', '', '', 0, "Fedora-Modular",

               fedfind.release.RawhideModularNightly, 'Rawhide', '', 'datestr', '0'),

+             ('Bikeshed', '', '', 0, "Fedora-Modular",

+              fedfind.release.BikeshedModularNightly, 'Bikeshed', '', 'datestr', '0'),

              (24, 'Atomic', '', 0, "Fedora", fedfind.release.AtomicNightly, '24', 'Atomic', 'datestr', '0'),

              (25, 'Docker', '', 0, "Fedora", fedfind.release.DockerNightly, '25', 'Docker', 'datestr', '0'),

              (25, 'Cloud', '', 0, "Fedora", fedfind.release.CloudNightly, '25', 'Cloud', 'datestr', '0'),
@@ -438,6 +440,18 @@ 

          assert ret.compose == '20170816'

          assert ret.respin == '0'

  

+     def test_get_release_url_cid_bikeshed_modular(self):

+         """Getting a release based on a correct Bikeshed modular

+         nightly URL should work.

+         """

+         url = ('https://kojipkgs.fedoraproject.org/'

+                'compose/Fedora-Modular-Bikeshed-20171013.n.0/compose/')

+         ret = fedfind.release.get_release(url=url)

+         assert isinstance(ret, fedfind.release.BikeshedModularNightly)

+         assert ret.release == 'Bikeshed'

+         assert ret.compose == '20171013'

+         assert ret.respin == '0'

+ 

      def test_get_release_url_cid_branched_modular(self):

          """Getting a release based on a correct Branched modular

          nightly URL should work.
@@ -550,6 +564,7 @@ 

  

      @mock.patch('fedfind.release.RawhideNightly', FakeRelease)

      @mock.patch('fedfind.release.RawhideModularNightly', FakeRelease)

+     @mock.patch('fedfind.release.BikeshedModularNightly', FakeRelease)

      @mock.patch('fedfind.release.BranchedNightly', FakeRelease)

      @mock.patch('fedfind.release.BranchedModularNightly', FakeRelease)

      @mock.patch('fedfind.release.AtomicNightly', FakeRelease)
@@ -572,6 +587,9 @@ 

          got = fedfind.release.get_release('Rawhide', '', '20170816', dist='Fedora-Modular')

          assert isinstance(got, fedfind.release.RawhideModularNightly)

          assert got.respin == 3

+         got = fedfind.release.get_release('Bikeshed', '', '20171013', dist='Fedora-Modular')

+         assert isinstance(got, fedfind.release.BikeshedModularNightly)

+         assert got.respin == 3

          got = fedfind.release.get_release(24, 'Atomic', '20160628')

          assert isinstance(got, fedfind.release.AtomicNightly)

          assert got.respin == 3
@@ -661,10 +679,13 @@ 

          # same release

          got2 = fedfind.release.get_release(cid=got.cid)

          assert got.version == got2.version

+         # getting a release with the old form of fake compose ID also

+         got3 = fedfind.release.get_release(cid='FedoraRespin-24-20161120.0')

+         assert got.version == got3.version

          # getting a release from the URL should get the same release

-         got3 = fedfind.release.get_release(

+         got4 = fedfind.release.get_release(

              url='https://dl.fedoraproject.org/pub/alt/live-respins/')

-         assert got3.version == got2.version

+         assert got4.version == got3.version

          # trying to get RespinRelease with a non-matching release

          # or compose should raise an error

          with pytest.raises(ValueError):
@@ -674,7 +695,7 @@ 

          with pytest.raises(ValueError):

              fedfind.release.get_release('25', 'Respin', '20170305')

          with pytest.raises(ValueError):

-             fedfind.release.get_release(cid='FedoraRespin-25-20170305.0')

+             fedfind.release.get_release(cid='FedoraRespin-25-updates-20170305.0')

  

      @pytest.mark.parametrize(

          ("release", "milestone", "compose", "respin", "cid"),

Merge the new compose ID parser, handle modular nightly composes from master being called Bikeshed not Rawhide now, and explicitly don't support updates and updates-testing composes.

Pull-Request has been closed by adamwill

6 years ago