#6860 Update fedretire to use PDC and pagure APIs.
Merged 4 years ago by kevin. Opened 6 years ago by ralph.
ralph/releng update-fedretire  into  master

file modified
+66 -14
@@ -9,7 +9,23 @@ 

  import subprocess

  import tempfile

  

- import pkgdb2client

+ import requests

+ import pdc_client

+ 

+ NAMESPACES = {

+     'rpms': 'rpm',

+     'modules': 'module',

+     'container': 'container',

+ }

+ 

+ RETIRED = 'Retired'

+ ORPHANED = 'Orphaned'

+ ACTIVE = 'Active'

+ 

+ # This is a user in pagure.

+ ORPHAN_UID = 'orphan'

+ 

+ PAGURE_URL = 'https://src.fedoraproject.org'

  

  ORPHAN_REASON = """Retired orphaned package, because it was orphaned for

  more than six weeks.
@@ -31,17 +47,51 @@ 

      "el6": ""

  }

  

- def get_status(name, branch="master"):

-     pkgdb = pkgdb2client.PkgDB()

-     response = pkgdb.get_package(name, branches=branch)

- 

-     for p in response.get("packages", []):

-         if p.get("package", {}).get("name") == name:

-             status = p.get("status")

-             return status

-     return ""

- 

- def check_status(name, branch, status="Orphaned", dep_status="Retired"):

+ def get_status(name, branch="master", namespace='rpms'):

+     # First, check to see if it is retired.

+     # PDCClient pulls connection information from /etc/pdc.d/

+     # develop=True means: don't authenticate.

+     pdc = pdc_client.PDCClient('fedora', develop=True)

+     kwargs = dict(global_component=name, name=branch, type=NAMESPACES[namespace])

+     results = pdc.get_paged(pdc['component-branches'], **kwargs)

+     try:

+         pdc_branch = results.next()

+     except StopIteration:

+         raise ValueError("branch %r of package %r does not exist in PDC." % (

+             branch, name))

+     retired = not pdc_branch['active']

+     if retired:

+         return RETIRED

+ 

+     # OK, if it isn't retired, it *might* be orphaned.. but we have to check

+     # pagure for that.

+     kwargs = dict(name=name, namespace=namespace)

+     response = requests.get(PAGURE_URL + "/api/0/packages", params=kwargs)
till commented 6 years ago

The branch is not considered to check if a package is orphaned.

That is a good point but as I understand it, Pagure over dist-git has no concept of a specific branch being orphaned. It's basically all or nothing, since a project is marked as orphaned only when the "main admin" on the project is the "orphan" user. Specific branches can be retired, and those are marked in PDC.

This is bad, what if something is changed in newer branches and the maintainer doesn't want to maintain a package anymore in newer branches or vice versa. How can we handle these kind of situations?

Any branches that aren't maintained would be marked as inactive in PDC by having their SLs EOL to the day those branches are orphaned or retired.

Also, those branches would have dead.package files in them.

till commented 6 years ago

The worklflow used to be that maintainers can decide to orphan or retire packages. If the package is ok in general but the maintainer does not want to maintain a package anymore or if the maintainer is nonresponsive, the package is orphaned. This only marks the packages as being in search of a new maintainer. If the package is not useful anymore, the maintainer should retire it (only possible in EPEL, Rawhide and Branched (before the final freeze)). Alternatively releng (usually me) retires packages that are orphaned for more than 6 weeks.

@till, thank you for bringing that up. I forgot about the orphan period in my last comment.

@ralph may have a better idea, but here's my suggestion. In the event that a package is orphaned, perhaps we change the "main admin" to orphan in Pagure, and keep the SLs on the current Fedora release branches (e.g. f26) the same. All other branches could be set to EOL in 6 weeks if no one picks it up.

That is more or less the same workflow but with the different tools. Thoughts?

till commented 6 years ago

@mprahl, the thing is, that it seems to me that if the EPEL branches and the Fedora branches are maintained by different people, i.e.there is a bugzilla override for EPEL in https://pagure.io/releng/fedora-scm-requests/tree/master, the EPEL branches should not be considered orphaned when the Fedora maintainer orphans the package. Therefore checking only the main admin would not be enough.

Btw. I guess also something needs to remove bugzilla overrides from the fedora-scm-requests repo when a package is retired to make sure the information is consistent. Not sure where this would belong to.

+     if not bool(response):

+         raise IOError("%r gave a %r" % (response.request.url, response))

+     data = response.json()

+     if not data['projects']:

+         raise ValueError("%r found no pagure repos" % (response.request.url))

+     people = set()

+     for kind in ('access_groups', 'access_users'):

+         people.extend(data[kind].values())

+ 

+     if len(people) == 0:

+         raise ValueError("%r has no people.. not even the orphan user.  "

+                          "Should not be possible." % response.request.url)

+ 

+     # See if 'orphan' is in there

+     if ORPHAN_UID in people:

+         # Sanity check

+         if len(people) > 1:

+             raise ValueError("%r is both orphaned and not." % response.request.url)

+         return ORPHANED

+ 

+     # If it is neither Retired nor Orphaned, then it must be active.

+     return ACTIVE

+ 

+ 

+ def check_status(name, branch, status=ORPHANED, dep_status=RETIRED):

      current_status = get_status(name, branch)

      if current_status != status:

          return False
@@ -60,6 +110,7 @@ 

              return False

      return True

  

+ 

  def retire(pkg, branch, reason, dryrun=False):

      def run(cmd, cwd):

          if dryrun:
@@ -82,9 +133,10 @@ 

      finally:

          shutil.rmtree(tempdir)

  

+ 

  if __name__ == "__main__":

      parser = argparse.ArgumentParser(description="Helper to retire packages")

-     parser.add_argument("--branch", default="master", nargs="*",

+     parser.add_argument("--branch", default=["master"], nargs="*",

                          choices=BRANCH_HIERARCHY.keys())

      parser.add_argument("--dry-run", default=False, action="store_true")

      parser.add_argument("--reasonfile", default=None)
@@ -140,7 +192,7 @@ 

                            "branches".format(pkg, branch))

                      continue

              elif args.orphan_dependent:

-                 if not check_status(args.orphan_dependent, branch, "Retired"):

+                 if not check_status(args.orphan_dependent, branch, RETIRED):

                      print("Skipped pkg '{}' on '{}' because "

                            "package '{}' is not retired in one or more checked "

                            "branches".format(

This repoints fedretire to query PDC and pagure for the retired/orphaned
status of packages.

See also https://fedoraproject.org/wiki/Changes/ArbitraryBranching

Signed-off-by: Ralph Bean rbean@redhat.com

pkgdb has entered read-only mode. Any reviewers here?

1 new commit added

  • Fix url.
6 years ago

The branch is not considered to check if a package is orphaned.

I usually only use this to retire packages when they are orphaned long enough. So I can try it the next time packages are going to be retired. I am not sure if other people use this script as well currently. However it seems that checking whether a package is orphaned is more complex nowadays. I am wondering if it makes sense to bundle the code to get a list of orphaned/retired packages and to check this status for an individual package into a small module since similar code is required for fedretire.py, find_unblocked_orphans.py and block_retired.py.

That is a good point but as I understand it, Pagure over dist-git has no concept of a specific branch being orphaned. It's basically all or nothing, since a project is marked as orphaned only when the "main admin" on the project is the "orphan" user. Specific branches can be retired, and those are marked in PDC.

This is bad, what if something is changed in newer branches and the maintainer doesn't want to maintain a package anymore in newer branches or vice versa. How can we handle these kind of situations?

Any branches that aren't maintained would be marked as inactive in PDC by having their SLs EOL to the day those branches are orphaned or retired.

Also, those branches would have dead.package files in them.

The worklflow used to be that maintainers can decide to orphan or retire packages. If the package is ok in general but the maintainer does not want to maintain a package anymore or if the maintainer is nonresponsive, the package is orphaned. This only marks the packages as being in search of a new maintainer. If the package is not useful anymore, the maintainer should retire it (only possible in EPEL, Rawhide and Branched (before the final freeze)). Alternatively releng (usually me) retires packages that are orphaned for more than 6 weeks.

@till, thank you for bringing that up. I forgot about the orphan period in my last comment.

@ralph may have a better idea, but here's my suggestion. In the event that a package is orphaned, perhaps we change the "main admin" to orphan in Pagure, and keep the SLs on the current Fedora release branches (e.g. f26) the same. All other branches could be set to EOL in 6 weeks if no one picks it up.

That is more or less the same workflow but with the different tools. Thoughts?

@mprahl, the thing is, that it seems to me that if the EPEL branches and the Fedora branches are maintained by different people, i.e.there is a bugzilla override for EPEL in https://pagure.io/releng/fedora-scm-requests/tree/master, the EPEL branches should not be considered orphaned when the Fedora maintainer orphans the package. Therefore checking only the main admin would not be enough.

Btw. I guess also something needs to remove bugzilla overrides from the fedora-scm-requests repo when a package is retired to make sure the information is consistent. Not sure where this would belong to.

Shoot - I missed all the conversation here.

Note, here's a related PR for orphan management -> https://pagure.io/releng/pull-request/6949

I'm going to just merge this since it's better than what we have.

Additional PR's welcome to fix/adjust. Discussion about per branch ownership should probibly be at a higher level than this PR. :european_post_office:

Pull-Request has been merged by kevin

4 years ago
Metadata