#11044 critpath.py: also record critpath by groups in a JSON file
Merged 2 months ago by kevin. Opened 2 months ago by adamwill.
adamwill/releng critpath-groups  into  main

file modified
+37 -18
@@ -13,6 +13,8 @@ 

  

  import sys

  import argparse

+ from collections import defaultdict

+ import json

  import shutil

  from tempfile import mkdtemp

  import dnf
@@ -84,7 +86,7 @@ 

          conf.arch = "armv7hl"

      else:

          conf.arch = arch

-     packages = set()

+     packages = dict()

  

      try:

          # add a new repo requires an id, a conf object, and a baseurl
@@ -113,7 +115,7 @@ 

              base.group_install(group, ["mandatory", "default", "optional"], strict=False)

              # resolve the groups marked in base object

              base.resolve()

-             packages = packages.union(base.transaction.install_set)

+             packages[group] = base.transaction.install_set

  

          return packages

  
@@ -162,7 +164,13 @@ 

          "-o",

          "--output",

          default="critpath.txt",

-         help="name of file to write critpath list (%(default)s)",

+         help="name of file to write flat plaintext critpath list (%(default)s)",

+     )

+     parser.add_argument(

+         "-j",

+         "--jsonout",

+         default="critpath.json",

+         help="name of file to write grouped JSON critpath list (%(default)s)",

      )

      parser.add_argument(

          "-u",
@@ -191,11 +199,18 @@ 

      return parser.parse_args()

  

  

- def write_file(critpath, outpath):

+ def write_files(critpath, outpath, jsonout):

+     wrapped = {"rpm": critpath}

+     with open(jsonout, mode="w", encoding="utf-8") as jsonoutfh:

+         json.dump(wrapped, jsonoutfh, sort_keys=True, indent=4)

+     print(f"Wrote grouped critpath data to {jsonout}")

+     pkgs = set()

+     for grppkgs in critpath.values():

+         pkgs = pkgs.union(set(grppkgs))

      with open(outpath, mode="w", encoding="utf-8") as outfh:

-         for packagename in sorted(critpath):

+         for packagename in sorted(pkgs):

              outfh.write(packagename + "\n")

-     package_count = len(critpath)

+     package_count = len(pkgs)

      print(f"Wrote {package_count} items to {outpath}")

  

  
@@ -229,7 +244,7 @@ 

          print(f"Using alternate arch update URL {updatealturl}")

  

      # Do the critpath expansion for each arch

-     critpath = set()

+     critpath = defaultdict(set)

      for arch in check_arches + alternate_check_arches:

          urls = [baseurl, updateurl]

          if arch in alternate_check_arches:
@@ -240,22 +255,26 @@ 

          urls = [url for url in urls if url]

  

          print(f"Expanding critical path for {arch}")

-         pkgs = expand_dnf_critpath(urls, arch)

+         pkgdict = expand_dnf_critpath(urls, arch)

  

-         package_count = len(pkgs)

-         print(f"{package_count} packages for {arch}")

+         for (group, pkgs) in pkgdict.items():

+             package_count = len(pkgs)

+             print(f"{package_count} packages in {group} for {arch}")

  

-         if args.nvr:

-             critpath.update([nvr(pkg) for pkg in pkgs])

-         elif args.srpm:

-             critpath.update([get_source(pkg.sourcerpm) for pkg in pkgs])

-         else:

-             critpath.update([pkg.name for pkg in pkgs])

+             if args.nvr:

+                 critpath[group].update([nvr(pkg) for pkg in pkgs])

+             elif args.srpm:

+                 critpath[group].update([get_source(pkg.sourcerpm) for pkg in pkgs])

+             else:

+                 critpath[group].update([pkg.name for pkg in pkgs])

  

-         del pkgs

+         del pkgdict

          print()

+     # Turn sets back into lists (so we can JSON-dump them)

+     for group in critpath:

+         critpath[group] = sorted(critpath[group])

  

-     write_file(critpath, args.output)

+     write_files(critpath, args.output, args.jsonout)

  

  

  if __name__ == "__main__":

This makes critpath.py output a JSON file containing a record
of the calculated critical path for each individual critpath
group, as well as the flat text file containing the entire
combined critical path. The ultimate goal is to move Bodhi over
to tagging updates as critical path by group and also doing
critical path gating queries per group, to give testers more
information on exactly what an update is "critical" for, and
allow more flexibility in scheduling automated tests and gating.

Signed-off-by: Adam Williamson awilliam@redhat.com

rebased onto e734a00fdd80182ff8b78e4820cb53166a0a2cbd

2 months ago

rebased onto 621caa5

2 months ago

Pull-Request has been merged by kevin

2 months ago

in case you're wondering why I 'wrap' the dict, it kinda fits in with how Bodhi works. I'm poking at Bodhi ATM and kinda planning on adding to Bodhi the ability to just consume this file as-is. Bodhi theoretically supports critpath operations for different content types - not just rpm - and it makes logical sense to handle that this way.

Metadata