#9633 Add a script to "retire" a packager
Merged 4 months ago by humaton. Opened 4 months ago by pingou.
pingou/releng master  into  master

@@ -0,0 +1,297 @@ 

+ #!/usr/bin/python3

+ 

+ """

+ This script queries dist-git for all the packages a given packager maintains,

+ has commit or watches.

+ Package that the packager is the main admin are then orphaned. The packager is

+ then removed from all packages that they have commit for and their watch status

+ is reset on every packages that they are watching.

+ 

+ """

+ 

+ import argparse

+ import collections

+ import logging

+ import os

+ import sys

+ 

+ import requests

+ from requests.adapters import HTTPAdapter

+ from requests.packages.urllib3.util.retry import Retry

+ 

+ _log = logging.getLogger(__name__)

+ dist_git_base = "https://src.fedoraproject.org"

+ pagure_token = None

+ 

+ 

+ def retry_session():

+     session = requests.Session()

+     retry = Retry(

+         total=5,

+         read=5,

+         connect=5,

+         backoff_factor=0.3,

+         status_forcelist=(500, 502, 504),

+     )

+     adapter = HTTPAdapter(max_retries=retry)

+     session.mount("http://", adapter)

+     session.mount("https://", adapter)

+     return session

+ 

+ 

+ def setup_logging(log_level: int):

+     handlers = []

+ 

+     _log.setLevel(log_level)

+     # We want all messages logged at level INFO or lower to be printed to stdout

+     info_handler = logging.StreamHandler(stream=sys.stdout)

+     handlers.append(info_handler)

+ 

+     if log_level == logging.INFO:

+         # In normal operation, don't decorate messages

+         for handler in handlers:

+             handler.setFormatter(logging.Formatter("%(message)s"))

+ 

+     logging.basicConfig(level=log_level, handlers=handlers)

+ 

+ 

+ def get_arguments(args):

+     """ Load and parse the CLI arguments."""

+     parser = argparse.ArgumentParser(

+         description="Looks for the specified list of users what they "

+         "maintain or watch in dist-git.\nIf --retire is specified, all the ACL "

+         "the packager(s) have in dist-git will be removed. If they are main admins "

+         "of some packages, these packages will be orphaned. If they have commit "

+         "access on some packages, they will no longer have these access. If they "

+         "watch a package, their watch status will be reset. Note: the source of "

+         "information is refreshed hourly, so if you run the script twice with "

+         "`--retire` you may not see a difference here."

+     )

+     parser.add_argument(

+         dest="usernames", nargs="*", help="Names of the users to retire.",

+     )

+     parser.add_argument(

+         "--from-file",

+         dest="users_file",

+         help="Path to a file containing the users to check (one per line).",

+     )

+     parser.add_argument(

+         "--retire",

+         action="store_true",

+         default=False,

+         help="Retire the user(s) (ie: orphan, remove from ACL, reset watch)",

+     )

+     parser.add_argument(

+         "--api-token",

+         dest="pagure_token",

+         default=os.environ.get("PAGURE_TOKEN"),

+         help="Pagure token to use to interact with dist-git. It can also be set "

+         "via the PAGURE_TOKEN environment variable. (This script requires the "

+         "`modifyproject` ACL to work)",

+     )

+     report_group = parser.add_mutually_exclusive_group()

+     report_group.add_argument(

+         "--watch",

+         action="store_const",

+         dest="report",

+         const="watch",

+         default="all",

+         help="Only report/act on watched projects",

+     )

+     report_group.add_argument(

+         "--maintain",

+         action="store_const",

+         dest="report",

+         const="maintain",

+         help="Only report/act projects the packagers have commit access to",

+     )

+ 

+     log_level_group = parser.add_mutually_exclusive_group()

+     log_level_group.add_argument(

+         "--debug",

+         action="store_const",

+         dest="log_level",

+         const=logging.DEBUG,

+         default=logging.INFO,

+         help="Enable debugging output",

+     )

+ 

+     return parser.parse_args(args)

+ 

+ 

+ def user_access(session, username, namespace_name):

+     """ Returns whether the specified username is listed in the maintainers

+     list of the specified package. """

+     req = session.get(f"{dist_git_base}/api/0/{namespace_name}")

+     project = req.json()

+     level = None

+     if username == project["user"]["name"]:

+         level = "main admin"

+     else:

+         maintainers = set()

+         for acl in project["access_users"]:

+             maintainers.update(set(project["access_users"][acl]))

+ 

+         if username in maintainers:

+             level = "maintainer"

+ 

+     return level

+ 

+ 

+ def unwatch_package(namespace, name, username):

+     """ Reset the watch status of the given user on the specified project. """

+     _log.debug("Going to reset watch status of %s on %s/%s", username, namespace, name)

+     base_url = dist_git_base.rstrip("/")

+     session = retry_session()

+ 

+     # Reset the watching status

+     url = f"{base_url}/api/0/{namespace}/{name}/watchers/update"

+     headers = {"Authorization": f"token {pagure_token}"}

+     data = {"status": -1, "watcher": username}

+ 

+     req = session.post(url, data=data, headers=headers)

+     if not req.ok:

+         print("**** REQUEST FAILED")

+         print("  - Unwatch package")

+         print(req.url)

+         print(data)

+         print(headers)

+         print(req.text)

+     else:

+         print(f"  {username} is no longer watching {namespace}/{name}")

+     session.close()

+ 

+ 

+ def orphan_package(namespace, name, username):

+     """ Give the specified project on dist_git to the ``orphan`` user.

+     """

+     _log.debug("Going to orphan: %s/%s from %s", namespace, name, username)

+     base_url = dist_git_base.rstrip("/")

+     session = retry_session()

+ 

+     # Orphan the package

+     url = f"{base_url}/api/0/{namespace}/{name}"

+     headers = {"Authorization": f"token {pagure_token}"}

+     data = {"main_admin": "orphan", "retain_access": False}

+ 

+     req = session.patch(url, data=data, headers=headers)

+     if not req.ok:

+         print("**** REQUEST FAILED")

+         print("  - Orphan package")

+         print(req.url)

+         print(data)

+         print(req.text)

+     else:

+         print(f"  {username} is no longer the main admin of {namespace}/{name}")

+     session.close()

+ 

+     unwatch_package(namespace, name, username)

+ 

+ 

+ def remove_access(namespace, name, username, usertype):

+     """ Remove the ACL of the specified user/group on the specified project. """

+     _log.debug("Going to remove %s from %s/%s", username, namespace, name)

+     base_url = dist_git_base.rstrip("/")

+     session = retry_session()

+ 

+     # Remove ACL on the package

+     url = f"{base_url}/{namespace}/{name}/git/modifyacls"

+     headers = {"Authorization": f"token {pagure_token}"}

+     data = {

+         "user_type": usertype,

+         "name": username,

+     }

+ 

+     req = session.patch(url, data=data, headers=headers)

+     if not req.ok:

+         print("**** REQUEST FAILED")

+         print("  - Remove ACL")

+         print(req.url)

+         print(data)

+         print(req.text)

+     else:

+         print(f"  {username} is no longer maintaining {namespace}/{name}")

+     session.close()

+ 

+     if usertype == "user":

+         # Reset the watching status

+         unwatch_package(namespace, name, username)

+ 

+ 

+ def main(args):

+     """ For the specified list of users, retrieve what they are maintaining

+     or watching in dist-git."""

+ 

+     args = get_arguments(args)

+     setup_logging(log_level=args.log_level)

+     _log.debug("Log level set to: %s", args.log_level)

+ 

+     if args.pagure_token:

+         global pagure_token

+         pagure_token = args.pagure_token

+ 

+     if not pagure_token and args.retire:

+         print(

+             "No pagure token set in the CLI argument or via the PAGURE_TOKEN "

+             "environment variable. Going to ignore --retire"

+         )

+         args.retire = False

+ 

+     usernames = []

+     if args.users_file:

+         _log.debug("Loading usernames for file: %s", args.users_file)

+         if not os.path.exists(args.users_file):

+             _log.info("No such file found: %s", args.users_file)

+         try:

+             with open(args.users_file) as stream:

+                 usernames = [l.strip() for l in stream.readlines()]

+         except Exception as err:

+             _log.debug(

+                 "Failed to load/read the file: %s, error is: %s", args.users_file, err

+             )

+     else:

+         _log.debug("Loading usernames for the CLI arguments")

+         usernames = args.usernames

+ 

+     _log.debug("Loading info from dist-git's pagure_bz.json file")

+     session = retry_session()

+     req = session.get(f"{dist_git_base}/extras/pagure_bz.json")

+     pagure_bz = req.json()

+     session.close()

+ 

+     packages_per_user = collections.defaultdict(list)

+     for namespace in pagure_bz:

+         for package in pagure_bz[namespace]:

+             _log.debug("Processing %s/%s", namespace, package)

+             for user in pagure_bz[namespace][package]:

+                 if user in usernames:

+                     packages_per_user[user].append(f"{namespace}/{package}")

+ 

+     for username in sorted(usernames):

+         _log.debug("Processing user: %s", username)

+         for pkg in sorted(packages_per_user[username]):

+             level = user_access(session, username, pkg)

+             if level:

+                 if args.report in ["all", "maintain"]:

+                     print(f"{username} is {level} of {pkg}")

+                     if args.retire:

+                         namespace, name = pkg.split("/", 1)

+                         if level == "main admin":

+                             orphan_package(namespace, name, username)

+                         elif level == "maintainer":

+                             remove_access(namespace, name, username, "user")

+ 

+             else:

+                 if args.report in ["all", "watch"]:

+                     print(f"{username} is watching {pkg}")

+                     if args.retire:

+                         namespace, name = pkg.split("/", 1)

+                         unwatch_package(namespace, name, username)

+         print()

+ 

+ 

+ if __name__ == "__main__":

+     try:

+         sys.exit(main(sys.argv[1:]))

+     except KeyboardInterrupt:

+         pass

@@ -1,151 +0,0 @@ 

- #!/usr/bin/python

- 

- """

- Simple small CLI utility to list who maintains/watches what on dist-git.

- 

- It can works from either one or more username specified as CLI argument

- or retrieve this list of usernames from a file that has one username per

- line.

- 

- It can be tweaked to report only packages watched, or maintained, or both

- (which is the default).

- 

- """

- 

- import argparse

- import collections

- import logging

- import os

- import sys

- 

- import requests

- 

- _log = logging.getLogger(__name__)

- dist_git_base = "https://src.fedoraproject.org"

- 

- 

- def setup_logging(log_level: int):

-     handlers = []

- 

-     _log.setLevel(log_level)

-     # We want all messages logged at level INFO or lower to be printed to stdout

-     info_handler = logging.StreamHandler(stream=sys.stdout)

-     handlers.append(info_handler)

- 

-     if log_level == logging.INFO:

-         # In normal operation, don't decorate messages

-         for handler in handlers:

-             handler.setFormatter(logging.Formatter("%(message)s"))

- 

-     logging.basicConfig(level=log_level, handlers=handlers)

- 

- 

- def get_arguments(args):

-     """ Load and parse the CLI arguments."""

-     parser = argparse.ArgumentParser(

-         description="Looks for the specified list of users what they "

-         "maintain or watch in dist-git."

-     )

-     parser.add_argument(

-         dest="usernames", nargs="*", help="Names of the users to check.",

-     )

-     parser.add_argument(

-         "--from-file",

-         dest="users_file",

-         help="Path to a file containing the users to check (one per line).",

-     )

-     report_group = parser.add_mutually_exclusive_group()

-     report_group.add_argument(

-         "--watch",

-         action="store_const",

-         dest="report",

-         const="watch",

-         default="all",

-         help="Only report watched projects",

-     )

-     report_group.add_argument(

-         "--maintain",

-         action="store_const",

-         dest="report",

-         const="maintain",

-         help="Only report maintained projects",

-     )

- 

-     log_level_group = parser.add_mutually_exclusive_group()

-     log_level_group.add_argument(

-         "--debug",

-         action="store_const",

-         dest="log_level",

-         const=logging.DEBUG,

-         default=logging.INFO,

-         help="Enable debugging output",

-     )

- 

-     return parser.parse_args(args)

- 

- 

- def is_maintainer(username, namespace_name):

-     """ Returns whether the specified username is listed in the maintainers

-     list of the specified package. """

-     req = requests.get(f"{dist_git_base}/api/0/{namespace_name}")

-     project = req.json()

-     maintainers = set([project["user"]["name"]])

-     for acl in project["access_users"]:

-         maintainers.update(set(project["access_users"][acl]))

- 

-     return username in maintainers

- 

- 

- def main(args):

-     """ For the specified list of users, retrieve what they are maintaining

-     or watching in dist-git."""

- 

-     args = get_arguments(args)

-     setup_logging(log_level=args.log_level)

-     _log.debug("Log level set to: %s", args.log_level)

- 

-     usernames = []

-     if args.users_file:

-         _log.debug("Loading usernames for file: %s", args.users_file)

-         if not os.path.exists(args.users_file):

-             _log.info("No such file found: %s", args.users_file)

-         try:

-             with open(args.users_file) as stream:

-                 usernames = [l.strip() for l in stream.readlines()]

-         except Exception as err:

-             _log.debug(

-                 "Failed to load/read the file: %s, error is: %s", args.users_file, err

-             )

-     else:

-         _log.debug("Loading usernames for the CLI arguments")

-         usernames = args.usernames

- 

-     _log.debug("Loading info from dist-git's pagure_bz.json file")

-     req = requests.get(f"{dist_git_base}/extras/pagure_bz.json")

-     pagure_bz = req.json()

- 

-     packages_per_user = collections.defaultdict(list)

-     for namespace in pagure_bz:

-         for package in pagure_bz[namespace]:

-             _log.debug("Processing %s/%s", namespace, package)

-             for user in pagure_bz[namespace][package]:

-                 if user in usernames:

-                     packages_per_user[user].append(f"{namespace}/{package}")

- 

-     for username in sorted(usernames):

-         _log.debug("Processing: %s", username)

-         for pkg in sorted(packages_per_user[username]):

-             if is_maintainer(username, pkg):

-                 if args.report in ["all", "maintain"]:

-                     print(f"{username} is maintaining {pkg}")

-             else:

-                 if args.report in ["all", "watch"]:

-                     print(f"{username} is watching {pkg}")

-         print()

- 

- 

- if __name__ == "__main__":

-     try:

-         sys.exit(main(sys.argv[1:]))

-     except KeyboardInterrupt:

-         pass

Basically, this script lists who maintains what and if given the
--retire argument, it will orphan all the packages for which the
packager(s) is/are main admin, it will remove the ACL for the
packager(s) on packages they have commit on and will reset their watch
status for everything they watch.

This script can be used when someone goes MIA and we want to mass-orphan
their packages. It can also be used when an account no longer has a valid
bugzilla account associated with their email in FAS.

Finally, the script can also only lists/acts on packages the packagers
maintain (via the --maintain argument) or only packages they watch
(via the --watch argument). Allowing to better tune the action
performed.

The second commit removes the who_maintains_what script whose
functionality is included in distgit/retire_packagers.py.

rebased onto e221c00412aac2f79666df1c4a5d3553bad7e2ef

4 months ago

rebased onto 0a2f00ae2b2674c22cbe9fb26c46d556beea9654

4 months ago

rebased onto 967a3a5

4 months ago
$ python retire_packagers.py pingou
pingou is main admin of rpms/R2spec
pingou is watching rpms/bzr-fastimport
pingou is main admin of rpms/dummy-test-package-crested
pingou is main admin of rpms/dummy-test-package-gloster
pingou is main admin of rpms/dummy-test-package-rubino
pingou is main admin of rpms/fedora-gather-easyfix
pingou is maintainer of rpms/fedora-review
pingou is maintainer of rpms/geany
Retrying (Retry(total=4, connect=5, read=4, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))': /api/0/rpms/geany-plugins
pingou is maintainer of rpms/geany-plugins
pingou is watching rpms/geanyvc
pingou is main admin of rpms/guake
pingou is maintainer of rpms/homebank
pingou is watching rpms/libdivecomputer
Retrying (Retry(total=4, connect=5, read=4, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))': /api/0/rpms/maven-resources-plugin
pingou is maintainer of rpms/maven-resources-plugin
pingou is maintainer of rpms/mirrormanager
pingou is main admin of rpms/pagure
pingou is main admin of rpms/pagure-dist-git
pingou is maintainer of rpms/pass
pingou is main admin of rpms/primer3
pingou is maintainer of rpms/python-GeoIP
pingou is main admin of rpms/python-arrow
pingou is main admin of rpms/python-bcrypt
pingou is maintainer of rpms/python-billiard
pingou is main admin of rpms/python-binaryornot
pingou is maintainer of rpms/python-case
pingou is maintainer of rpms/python-celery
pingou is maintainer of rpms/python-chai
pingou is main admin of rpms/python-contextlib2
pingou is main admin of rpms/python-flask-multistatic
pingou is main admin of rpms/python-flask-wtf
pingou is main admin of rpms/python-flask-xml-rpc
pingou is maintainer of rpms/python-grokmirror
pingou is maintainer of rpms/python-hypothesis
pingou is maintainer of rpms/python-igraph
pingou is maintainer of rpms/python-kitchen
pingou is maintainer of rpms/python-kombu
pingou is maintainer of rpms/python-nose-cover3
pingou is maintainer of rpms/python-pygit2
pingou is maintainer of rpms/python-rdflib
pingou is main admin of rpms/python-schema
pingou is main admin of rpms/python-sqlalchemy-utils
pingou is main admin of rpms/python-straight-plugin
pingou is main admin of rpms/python-summershum
pingou is main admin of rpms/python-trollius-redis
pingou is maintainer of rpms/python-vine
pingou is main admin of rpms/python-vobject
pingou is main admin of rpms/python-watchdog
pingou is maintainer of rpms/python-wtforms
pingou is maintainer of rpms/python-xlrd
pingou is watching rpms/repo_manager

Pull-Request has been merged by humaton

4 months ago

@churchyard hm, is that a problem? My understanding is that this is pagure API behavior.

I am testing this further. Could the token be read from a "standard" location? AFAIK give-package.py reads it from ~/.config/fedscm-admin/config.ini somehow.

I assume this is a missing scope problem?

$ PAGURE_TOKEN=... python retire_packagers.py --from-file nonresponsives --retire
avesh is maintainer of rpms/ima-evm-utils
**** REQUEST FAILED
  - Remove ACL
https://src.fedoraproject.org/rpms/ima-evm-utils/git/modifyacls
{'user_type': 'user', 'name': 'avesh'}
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<title>404 Not Found</title>
<h1>Not Found</h1>
<p>The requested URL was not found on the server.  If you entered the URL manually please check your spelling and try again.</p>

  avesh is no longer watching rpms/ima-evm-utils
avesh is maintainer of rpms/libtnc
**** REQUEST FAILED
  - Remove ACL
https://src.fedoraproject.org/rpms/libtnc/git/modifyacls
{'user_type': 'user', 'name': 'avesh'}
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<title>404 Not Found</title>
<h1>Not Found</h1>
<p>The requested URL was not found on the server.  If you entered the URL manually please check your spelling and try again.</p>

  avesh is no longer watching rpms/libtnc
avesh is watching rpms/openpts
  avesh is no longer watching rpms/openpts
avesh is watching rpms/openswan

OTOH, the token has "Modify an existing project" ACL.

$ PAGURE_TOKEN=... python retire_packagers.py --from-file nonresponsives --retire
avesh is maintainer of rpms/ima-evm-utils
**** REQUEST FAILED
  - Remove ACL

I actually fixed this in a subsequent push, the version that has been merged shouldn't have this problem

I actually fixed this in a subsequent push, the version that has been merged shouldn't have this problem

Apparently nope, submitting a new PR for this