fedora-qa / fedfind

Clone
Source Code
GIT

#18 Use Bodhi not (orphaned) pkgdb for current release info (#17)
Merged 4 years ago by adamwill. Opened 4 years ago by adamwill.
pkgdb-bodhi  into  master

file modified
+34 -27 
@@ -71,7 +71,7 @@ 
 

  

  # cache for pkgdb collections JSON, as there are cases where we want
 

  # to re-use it
 

- COLLECTIONS = None
 

+ RELEASES = None
 

  

  def date_check(value, fail_raise=True, out='obj'):
 

      """Checks whether a value is a date, and returns it if so. Valid
 
@@ -164,52 +164,59 @@ 
 

      """
 

      return string.lower().split(',')
 

  

- def _get_collections():
 

-     """Download the collections JSON if we haven't already, and cache
 

-     it at the module level. Expires after one day (so we don't have to
 

-     restart consumers etc. when we cut a release or branch).
 

+ def _get_releases():
 

+     """Download the Bodhi 'releases' JSON if we haven't already, and
 

+     cache it at the module level. Expires after one day (so we don't
 

+     have to restart consumers etc. when we cut a release or branch).
 

      """
 

      # yes, pylint, we really want a global. sheesh.
 

      # pylint:disable=global-statement
 

-     global COLLECTIONS
 

+     global RELEASES
 

  

-     if COLLECTIONS:
 

-         # check if we should expire the cache. first item in COLLECTIONS
 

+     if RELEASES:
 

+         # check if we should expire the cache. first item in RELEASES
 

          # is the datetime it was last updated
 

-         delta = datetime.datetime.utcnow() - COLLECTIONS[0]
 

+         delta = datetime.datetime.utcnow() - RELEASES[0]
 

          # this means "if it was more than a day ago"
 

          if delta > datetime.timedelta(1):
 

-             COLLECTIONS = []
 

+             RELEASES = []
 

  

-     if not COLLECTIONS:
 

-         logger.debug("collections not cached or cache expired, downloading...")
 

-         gotjson = download_json('https://admin.fedoraproject.org/pkgdb/api/collections/')
 

-         COLLECTIONS = [datetime.datetime.utcnow(), gotjson['collections']]
 

+     if not RELEASES:
 

+         logger.debug("releases not cached or cache expired, downloading...")
 

+         # wonder if I'll be gone before we start paginating here!
 

+         gotjson = download_json('https://bodhi.fedoraproject.org/releases/?rows_per_page=500')
 

+         RELEASES = [datetime.datetime.utcnow(), gotjson['releases']]
 

  

      # only return the actual data, not the timestamp...
 

-     return COLLECTIONS[1]
 

+     return RELEASES[1]
 

  

  def get_current_release(branched=False):
 

      """Find out what the 'current' Fedora release is, according to
 

-     the pkgdb API. If 'branched' is truth-y, will return the Branched
 

-     release number if there is one.
 

+     the Bodhi 'releases' API. If 'branched' is truth-y, will return
 

+     the Branched release number if there is one.
 

      """
 

-     collections = _get_collections()
 

+     releases = _get_releases()
 

+     # find current releases
 

+     rels = {int(rel['version']) for rel in releases if
 

+             rel['state'] == 'current' and rel['id_prefix'] == 'FEDORA'}
 

      if branched:
 

-         rels = [coll['version'] for coll in collections if
 

-                 coll['status'] in ('Active', 'Under Development')]
 

-     else:
 

-         rels = [coll['version'] for coll in collections if coll['status'] == 'Active']
 

-     rels = [int(rel) for rel in rels if rel.isdigit()]
 

+         # find development releases
 

+         devrels = {int(rel['version']) for rel in releases if
 

+                    rel['state'] == 'pending' and rel['id_prefix'] == 'FEDORA'}
 

+         # if there is only one development release, it's rawhide, and
 

+         # we should ignore it. if there's more than one, we'll assume
 

+         # the *lowest numbered* is Branched
 

+         if len(devrels) > 1:
 

+             rels.add(min(devrels))
 

      return max(rels)
 

  

  def get_current_stables():
 

      """Returns a list of the release numbers of all current stable
 

-     releases (according to pkgdb).
 

+     releases (according to Bodhi).
 

      """
 

-     collections = _get_collections()
 

-     return [int(coll['version']) for coll in collections if
 

-             coll['status'] == 'Active' and coll['name'] == 'Fedora']
 

+     releases = _get_releases()
 

+     return [int(rel['version']) for rel in releases if
 

+             rel['state'] == 'current' and rel['id_prefix'] == 'FEDORA']
 

  

  def _type_from_supported(types, filename):
 

      """We run through this twice, with slightly different candidate

file modified
+214 -45 
@@ -41,54 +41,215 @@ 
 

  import fedfind.const
 

  import fedfind.helpers
 

  

- # Fake data for the collections fixture
 

- COLLECTIONS_JSON = [
 

+ # Fake data for the releases fixtures. This is pruned real data as of
 

+ # 2020-02-21, but with F33 (Rawhide) removed, to act as if we had no
 

+ # Branched, only stable F31 and Rawhide F32. We include samples of
 

+ # all the other release types the functions should basically ignore.
 

+ RELEASES_NOBRANCHED_JSON = [
 

      {
 

-         "allow_retire": True,
 

-         "branchname": "master",
 

-         "date_created": "2014-05-14 12:36:15",
 

-         "date_updated": "2016-02-23 22:56:34",
 

-         "dist_tag": ".fc25",
 

-         "koji_name": "rawhide",
 

-         "name": "Fedora",
 

-         "status": "Under Development",
 

-         "version": "devel"
 

+         "branch": "el6",
 

+         "candidate_tag": "dist-6E-epel-testing-candidate",
 

+         "composed_by_bodhi": True,
 

+         "composes": [],
 

+         "create_automatic_updates": None,
 

+         "dist_tag": "dist-6E-epel",
 

+         "id_prefix": "FEDORA-EPEL",
 

+         "long_name": "Fedora EPEL 6",
 

+         "mail_template": "fedora_epel_legacy_errata_template",
 

+         "name": "EL-6",
 

+         "override_tag": "dist-6E-epel-override",
 

+         "package_manager": "unspecified",
 

+         "pending_signing_tag": "",
 

+         "pending_stable_tag": "",
 

+         "pending_testing_tag": "",
 

+         "stable_tag": "dist-6E-epel",
 

+         "state": "current",
 

+         "testing_repository": None,
 

+         "testing_tag": "dist-6E-epel-testing",
 

+         "version": "6"
 

      },
 

      {
 

-         "allow_retire": False,
 

-         "branchname": "f22",
 

-         "date_created": "2015-02-10 14:00:01",
 

-         "date_updated": "2015-02-10 14:00:01",
 

-         "dist_tag": ".fc22",
 

-         "koji_name": "f22",
 

-         "name": "Fedora",
 

-         "status": "Active",
 

-         "version": "22"
 

+         "branch": "f30m",
 

+         "candidate_tag": "f30-modular-updates-candidate",
 

+         "composed_by_bodhi": True,
 

+         "composes": [],
 

+         "create_automatic_updates": None,
 

+         "dist_tag": "f30-modular",
 

+         "id_prefix": "FEDORA-MODULAR",
 

+         "long_name": "Fedora 30 Modular",
 

+         "mail_template": "fedora_errata_template",
 

+         "name": "F30M",
 

+         "override_tag": "f30-modular-override",
 

+         "package_manager": "unspecified",
 

+         "pending_signing_tag": "f30-modular-signing-pending",
 

+         "pending_stable_tag": "f30-modular-updates-pending",
 

+         "pending_testing_tag": "f30-modular-updates-testing-pending",
 

+         "stable_tag": "f30-modular-updates",
 

+         "state": "current",
 

+         "testing_repository": None,
 

+         "testing_tag": "f30-modular-updates-testing",
 

+         "version": "30"
 

      },
 

      {
 

-         "allow_retire": False,
 

-         "branchname": "f23",
 

-         "date_created": "2015-07-14 18:13:12",
 

-         "date_updated": "2015-07-14 18:13:12",
 

-         "dist_tag": ".fc23",
 

-         "koji_name": "f23",
 

-         "name": "Fedora",
 

-         "status": "Active",
 

-         "version": "23"
 

+         "branch": "f32",
 

+         "candidate_tag": "f32-container-updates-candidate",
 

+         "composed_by_bodhi": True,
 

+         "composes": [],
 

+         "create_automatic_updates": False,
 

+         "dist_tag": "f32-container",
 

+         "id_prefix": "FEDORA-CONTAINER",
 

+         "long_name": "Fedora 32 Containers",
 

+         "mail_template": "fedora_errata_template",
 

+         "name": "F32C",
 

+         "override_tag": "f32-container-override",
 

+         "package_manager": "unspecified",
 

+         "pending_signing_tag": "",
 

+         "pending_stable_tag": "f32-container-updates-pending",
 

+         "pending_testing_tag": "f32-container-updates-testing-pending",
 

+         "stable_tag": "f32-container-updates",
 

+         "state": "pending",
 

+         "testing_repository": None,
 

+         "testing_tag": "f32-container-updates-testing",
 

+         "version": "32"
 

      },
 

      {
 

-         "allow_retire": True,
 

-         "branchname": "f24",
 

-         "date_created": "2016-02-23 22:57:55",
 

-         "date_updated": "2016-02-25 20:39:53",
 

-         "dist_tag": ".fc24",
 

-         "koji_name": "f24",
 

-         "name": "Fedora",
 

-         "status": "Under Development",
 

-         "version": "24"
 

+         "branch": "f31",
 

+         "candidate_tag": "f31-flatpak-updates-candidate",
 

+         "composed_by_bodhi": True,
 

+         "composes": [],
 

+         "create_automatic_updates": False,
 

+         "dist_tag": "f31-flatpak",
 

+         "id_prefix": "FEDORA-FLATPAK",
 

+         "long_name": "Fedora 31 Flatpaks",
 

+         "mail_template": "fedora_errata_template",
 

+         "name": "F31F",
 

+         "override_tag": "f31-flatpak-override",
 

+         "package_manager": "unspecified",
 

+         "pending_signing_tag": "",
 

+         "pending_stable_tag": "f31-flatpak-updates-pending",
 

+         "pending_testing_tag": "f31-flatpak-updates-testing-pending",
 

+         "stable_tag": "f31-flatpak-updates",
 

+         "state": "current",
 

+         "testing_repository": None,
 

+         "testing_tag": "f31-flatpak-updates-testing",
 

+         "version": "31"
 

+     },
 

+     {
 

+         "branch": "f21",
 

+         "candidate_tag": "f21-updates-candidate",
 

+         "composed_by_bodhi": True,
 

+         "composes": [],
 

+         "create_automatic_updates": None,
 

+         "dist_tag": "f21",
 

+         "id_prefix": "FEDORA",
 

+         "long_name": "Fedora 21",
 

+         "mail_template": "fedora_errata_template",
 

+         "name": "F21",
 

+         "override_tag": "f21-override",
 

+         "package_manager": "dnf",
 

+         "pending_signing_tag": "",
 

+         "pending_stable_tag": "f21-updates-pending",
 

+         "pending_testing_tag": "f21-updates-testing-pending",
 

+         "stable_tag": "f21-updates",
 

+         "state": "archived",
 

+         "testing_repository": "updates-testing",
 

+         "testing_tag": "f21-updates-testing",
 

+         "version": "21"
 

+     },
 

+     {
 

+         "branch": "f30",
 

+         "candidate_tag": "f30-updates-candidate",
 

+         "composed_by_bodhi": True,
 

+         "composes": [],
 

+         "create_automatic_updates": None,
 

+         "dist_tag": "f30",
 

+         "id_prefix": "FEDORA",
 

+         "long_name": "Fedora 30",
 

+         "mail_template": "fedora_errata_template",
 

+         "name": "F30",
 

+         "override_tag": "f30-override",
 

+         "package_manager": "dnf",
 

+         "pending_signing_tag": "f30-signing-pending",
 

+         "pending_stable_tag": "f30-updates-pending",
 

+         "pending_testing_tag": "f30-updates-testing-pending",
 

+         "stable_tag": "f30-updates",
 

+         "state": "current",
 

+         "testing_repository": "updates-testing",
 

+         "testing_tag": "f30-updates-testing",
 

+         "version": "30"
 

+     },
 

+     {
 

+         "branch": "f31",
 

+         "candidate_tag": "f31-updates-candidate",
 

+         "composed_by_bodhi": True,
 

+         "composes": [],
 

+         "create_automatic_updates": False,
 

+         "dist_tag": "f31",
 

+         "id_prefix": "FEDORA",
 

+         "long_name": "Fedora 31",
 

+         "mail_template": "fedora_errata_template",
 

+         "name": "F31",
 

+         "override_tag": "f31-override",
 

+         "package_manager": "dnf",
 

+         "pending_signing_tag": "f31-signing-pending",
 

+         "pending_stable_tag": "f31-updates-pending",
 

+         "pending_testing_tag": "f31-updates-testing-pending",
 

+         "stable_tag": "f31-updates",
 

+         "state": "current",
 

+         "testing_repository": "updates-testing",
 

+         "testing_tag": "f31-updates-testing",
 

+         "version": "31"
 

+     },
 

+     {
 

+         "branch": "f32",
 

+         "candidate_tag": "f32-updates-candidate",
 

+         "composed_by_bodhi": False,
 

+         "composes": [],
 

+         "create_automatic_updates": True,
 

+         "dist_tag": "f32",
 

+         "id_prefix": "FEDORA",
 

+         "long_name": "Fedora 32",
 

+         "mail_template": "fedora_errata_template",
 

+         "name": "F32",
 

+         "override_tag": "f32-override",
 

+         "package_manager": "dnf",
 

+         "pending_signing_tag": "f32-signing-pending",
 

+         "pending_stable_tag": "f32-updates-pending",
 

+         "pending_testing_tag": "f32-updates-testing-pending",
 

+         "stable_tag": "f32",
 

+         "state": "pending",
 

+         "testing_repository": "updates-testing",
 

+         "testing_tag": "f32-updates-testing",
 

+         "version": "32"
 

      }
 

  ]
 

  

+ # Now we copy it and add F33 back in, to give us data with a Branched
 

+ # and a Rawhide.
 

+ RELEASES_BRANCHED_JSON = list(RELEASES_NOBRANCHED_JSON)
 

+ RELEASES_BRANCHED_JSON.append({
 

+     "branch": "f33",
 

+     "candidate_tag": "f33-updates-candidate",
 

+     "composed_by_bodhi": False,
 

+     "composes": [],
 

+     "create_automatic_updates": True,
 

+     "dist_tag": "f33",
 

+     "id_prefix": "FEDORA",
 

+     "long_name": "Fedora 33",
 

+     "mail_template": "fedora_errata_template",
 

+     "name": "F33",
 

+     "override_tag": "f33-override",
 

+     "package_manager": "unspecified",
 

+     "pending_signing_tag": "f33-signing-pending",
 

+     "pending_stable_tag": "f33-updates-pending",
 

+     "pending_testing_tag": "f33-updates-testing-pending",
 

+     "stable_tag": "f33",
 

+     "state": "pending",
 

+     "testing_repository": None,
 

+     "testing_tag": "f33-updates-testing",
 

+     "version": "33"
 

+ })
 

+ 

  @pytest.fixture(scope="session")
 

  def http(request):
 

      """Run a SimpleHTTPServer that we can use as a fake dl.fp.o. Serve
 
@@ -182,20 +343,28 @@ 
 

          yield
 

  

  @pytest.fixture(scope="function")
 

- def collections(request):
 

-     """Mock fedfind.helpers._get_collections to return test data."""
 

-     with mock.patch('fedfind.helpers._get_collections', return_value=COLLECTIONS_JSON,
 

+ def releases_branched(request):
 

+     """Mock fedfind.helpers._get_releases to return test data (with a
 

+     Branched release)."""
 

+     with mock.patch('fedfind.helpers._get_releases', return_value=RELEASES_BRANCHED_JSON,
 

                      autospec=True):
 

          yield
 

  

  @pytest.fixture(scope="function")
 

- def json_collections(request):
 

-     """Mock fedfind.helpers.download_json to return collections test
 

+ def releases_nobranched(request):
 

+     """Mock fedfind.helpers._get_releases to return test data."""
 

+     with mock.patch('fedfind.helpers._get_releases', return_value=RELEASES_NOBRANCHED_JSON,
 

+                    autospec=True):
 

+         yield
 

+ 

+ @pytest.fixture(scope="function")
 

+ def json_releases(request):
 

+     """Mock fedfind.helpers.download_json to return releases test
 

      data (wrapped appropriately). Returns the mock so tests can check
 

      its call count, and the test data so they can verify it.
 

      """
 

      with mock.patch('fedfind.helpers.download_json',
 

-                     return_value={'collections': COLLECTIONS_JSON}, autospec=True) as mocked:
 

-         yield (mocked, COLLECTIONS_JSON)
 

+                     return_value={'releases': RELEASES_BRANCHED_JSON}, autospec=True) as mocked:
 

+         yield (mocked, RELEASES_BRANCHED_JSON)
 

  

  # vim: set textwidth=100 ts=8 et sw=4:

file modified
+19 -12 
@@ -206,31 +206,38 @@ 
 

          """It splits lists on commas. It ain't rocket science."""
 

          assert fedfind.helpers.comma_list('foo,Bar,moo') == ['foo', 'bar', 'moo']
 

  

-     def test_get_collections(self, json_collections):
 

-         """Test _get_collections works correctly and caches."""
 

-         (mocked, fakedata) = json_collections
 

+     def test_get_releases(self, json_releases):
 

+         """Test _get_releases works correctly and caches."""
 

+         (mocked, fakedata) = json_releases
 

          # download twice to check caching is working
 

          with freeze_time("2019-10-01"):
 

-             colls = fedfind.helpers._get_collections()
 

-             colls = fedfind.helpers._get_collections()
 

+             colls = fedfind.helpers._get_releases()
 

+             colls = fedfind.helpers._get_releases()
 

              assert colls == fakedata
 

              assert mocked.call_count == 1
 

          # now test the cache expiry works
 

          with freeze_time("2019-10-03"):
 

-             colls = fedfind.helpers._get_collections()
 

+             colls = fedfind.helpers._get_releases()
 

              assert colls == fakedata
 

              assert mocked.call_count == 2
 

  

-     def test_get_current_release(self, collections):
 

+     def test_get_current_release_branched(self, releases_branched):
 

          """Test get_current_release gives expected values, with our
 

-         fake collections data.
 

+         fake collections data (with a Branched release).
 

          """
 

-         assert fedfind.helpers.get_current_release() == 23
 

-         assert fedfind.helpers.get_current_release(branched=True) == 24
 

+         assert fedfind.helpers.get_current_release() == 31
 

+         assert fedfind.helpers.get_current_release(branched=True) == 32
 

  

-     def test_get_current_stables(self, collections):
 

+     def test_get_current_release_nobranched(self, releases_nobranched):
 

+         """Test get_current_release gives expected values, with our
 

+         fake collections data (with no Branched release).
 

+         """
 

+         assert fedfind.helpers.get_current_release() == 31
 

+         assert fedfind.helpers.get_current_release(branched=True) == 31
 

+ 

+     def test_get_current_stables(self, releases_branched):
 

          """Test get_current_stables gives the expected values."""
 

-         assert fedfind.helpers.get_current_stables() == [22, 23]
 

+         assert fedfind.helpers.get_current_stables() == [30, 31]
 

  

      @pytest.mark.parametrize(
 

          ("path", "arch", "form", "typ", "subvariant", "disc_number"),

Until now we've used
https://admin.fedoraproject.org/pkgdb/api/collections/ as our
source of truth for deciding what Fedora releases exist and what
state they're in (for the purposes of get_current_release()
and get_current_stables()). This used to be an API endpoint of
the pkgdb webapp; in 2017 pkgdb was retired, but because we and
GNOME Software (and possibly some other things) were still using
it, releng kept this specific URL active and serving a static
JSON-formatted text file that has to be updated by hand each time
we branch, cut a stable release, or EOL a release.

Unsurprisingly, this doesn't always happen perfectly on time, or
without mistakes (recently, F32 was added but initally marked as
'stable' rather than 'development', for instance). So to avoid
such problems, and hopefully to help releng eventually retire
this silly hack, let's port off it and use the Bodhi 'releases'
API instead. It provides all the information we need, and as
it's a real API of an important system, should be updated on time
and with correct data reliably.

Signed-off-by: Adam Williamson awilliam@redhat.com

rebased onto d04dc9c
4 years ago

rebased onto 058df5e
4 years ago

Pull-Request has been merged by adamwill
4 years ago
Changes Summary 3
+34 -27
file changed
fedfind/helpers.py
+214 -45
file changed
tests/conftest.py
+19 -12
file changed
tests/test_helpers.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.