#1291 cli: add script to list package build order in copr or koji
Merged 4 years ago by praiskup. Opened 4 years ago by dturecek.
copr/ dturecek/copr package-build-order  into  master

file modified
+3
@@ -32,6 +32,7 @@ 

  Requires:      python3-jinja2

  Requires:      python3-simplejson

  Requires:      python3-humanize

+ Requires:      python3-koji

  

  Recommends:    python3-progress

  
@@ -99,6 +100,7 @@ 

  install -d %{buildroot}%{_datadir}/cheat

  cp -a man/copr-cli.cheat %{buildroot}%{_datadir}/cheat/copr-cli

  ln -s %{_datadir}/cheat/copr-cli %{buildroot}%{_datadir}/cheat/copr

+ install -m 755 copr_cli/package_build_order.py %{buildroot}/%{_bindir}/package-build-order

  

  

  %check
@@ -117,6 +119,7 @@ 

  %{_datadir}/cheat/copr-cli

  %{_datadir}/cheat/copr

  %{python_sitelib}/*

+ %{_bindir}/package-build-order

  

  

  %changelog

@@ -0,0 +1,142 @@ 

+ #!/usr/bin/python3

+ """List packages of a given copr project in the order in which they were built."""

+ 

+ import os

+ import sys

+ import argparse

+ from configparser import ConfigParser

+ 

+ from copr.v3 import BuildProxy, BuildChrootProxy, config_from_file

+ from copr.v3.exceptions import CoprNoConfigException, CoprNoResultException, CoprRequestException

+ 

+ from koji import ClientSession, GenericError, BUILD_STATES

+ 

+ 

+ def package_order_from_copr(args):

+     """List package build order from Copr"""

+     if not args.project:

+         print("You need to specify Copr project to list package build order.")

+         sys.exit(1)

+ 

+     if not args.config:

+         args.config = "~/.config/copr"

+ 

+     try:

+         config_file = config_from_file(args.config)

+     except CoprNoConfigException:

+         print("Couldn't find copr config file at {0}.".format(args.config))

+         sys.exit(1)

+ 

+     try:

+         build_proxy = BuildProxy(config_file)

+         build_chroot_proxy = BuildChrootProxy(config_file)

+         project = args.project.split("/")

+         username = project[0]

+         projectname = project[1]

+         build_list = build_proxy.get_list(username, projectname)

+     except CoprNoResultException:

+         print("No copr project {0}/{1}.".format(username, projectname))

+         sys.exit(1)

+     except CoprRequestException:

+         print("Failed to get information from Copr.")

+         sys.exit(1)

+ 

+     build_list.reverse()

+     processed_packages = []

+     for build in build_list:

+         if args.chroot and args.chroot not in build["chroots"]:

+             continue

+         if build["state"] != "succeeded":

+             if not args.chroot:

+                 continue

+             if len(build["chroots"]) == 1:

+                 continue

+             build_chroot = build_chroot_proxy.get(build["id"], args.chroot)

+             if build_chroot.state != "succeeded":

+                 continue

+ 

+         package = "{0}-{1}".format(build["source_package"]["name"], build["source_package"]["version"])

+         if not package:

+             continue

+         if not args.list_each:

+             if package in processed_packages:

The --list-each is still not ideal; you probably want to skip per package name, not per NVR.

Or maybe not, there could be two options for two separate tasks. But the --list-all really deserves proper argparse doc; what it does precisely.

+                 continue

+             processed_packages.append(package)

+ 

+         print(package)

+ 

+ 

+ def package_order_from_koji(args):

+     """List package build order from Koji"""

+     if not args.username and not args.tag:

+         print("You need to specify either username or tag to list build order in Koji.")

+         sys.exit(1)

+ 

+     for path in [args.config, "~/.config/koji", "/etc/koji.conf"]:

+         if path and os.path.exists(path):

+             args.config = path

+ 

+     if not args.config:

+         print("Couldn't find koji config file.")

+ 

+     config_file = ConfigParser()

+     config_file.read(args.config)

+ 

+     koji_url = config_file.get("koji", "server")

+     session = ClientSession(koji_url)

+ 

+     if args.tag:

+         try:

+             builds = session.listTagged(tag=args.tag, owner=args.username)

+         except GenericError:

+             print("No tag {0}.".format(args.tag))

+             sys.exit(1)

+ 

+     elif args.username:

+         user = session.getUser(args.username)

+         if not user:

+             print("No user {0}.".format(args.username))

+             sys.exit(1)

+ 

+         user_id = user["id"]

+         builds = session.listBuilds(userID=user_id)

+ 

+     processed_packages = []

+     for build in sorted(builds, key=lambda i: i["completion_time"]):

+         if build["state"] != BUILD_STATES["COMPLETE"]:

+             continue

+         package = build["nvr"]

+         if not package:

+             continue

+         if not args.list_each:

+             if package in processed_packages:

+                 continue

+             processed_packages.append(package)

+ 

+         print(package)

+ 

+ 

+ if __name__ == "__main__":

+     parser = argparse.ArgumentParser()

+     parser.add_argument("--config", type=str, help="Path to copr/koji config")

+     parser.add_argument("--list-each", "-e", action="store_true", default=False,

+                         help="List each occurence of every package in the project")

+ 

+     subparsers = parser.add_subparsers(title="commands")

+ 

+     parser_copr = subparsers.add_parser("copr", help="List package build order in Copr")

+     parser_copr.add_argument("--project", "-p", type=str, help="Copr project in `owner/project` format")

+     parser_copr.add_argument("--chroot", "-c", type=str, help="List this chroot only")

+     parser_copr.set_defaults(func=package_order_from_copr)

+ 

+     parser_koji = subparsers.add_parser("koji", help="List package build order in Koji")

+     parser_koji.add_argument("--username", "-u", type=str, help="Koji username")

+     parser_koji.add_argument("--tag", "-t", type=str, help="Koji tag")

+     parser_koji.set_defaults(func=package_order_from_koji)

+ 

+     args = parser.parse_args()

+     try:

+         args.func(args)

+     except AttributeError:

+         parser.print_help()

+         sys.exit(0)

file modified
+9 -2
@@ -3,6 +3,7 @@ 

  import codecs

  import os

  import re

+ import shutil

  

  from setuptools import setup, find_packages

  
@@ -17,7 +18,8 @@ 

      'copr',

      'humanize',

      'simplejson',

-     'jinja2'

+     'jinja2',

+     'koji',

  ]

  

  __name__ = 'copr-cli'
@@ -27,6 +29,10 @@ 

  __url__ = "https://pagure.io/copr/copr"

  

  

+ if not os.path.exists('build/scripts'):

+     os.makedirs('build/scripts')

+ shutil.copyfile('copr_cli/package_build_order.py', 'build/scripts/package-build-order')

+ 

  setup(

      name=__name__,

      version="1.85",
@@ -49,7 +55,8 @@ 

      zip_safe=False,

      entry_points={

          'console_scripts': [

-             'copr-cli = copr_cli.main:main'

+             'copr-cli = copr_cli.main:main',

          ]

      },

+     scripts=['build/scripts/package-build-order'],

  )

no initial comment

The overall approach looks good to me, we may need some small adjustments though.

AFAIK we are not interested as much in the exact chronological order in which the packages were submitted, but we rather want to know in what order they can be successfully built (= the order they were successfully built in the past)

I have a project and submitted package foo into it. The build failed (was canceled, to be precise, but it doesn't change anything). Then I submitted bar which succeeded and then submitted foo again and it also succeeded this time. Let's say that the foo is dependent on bar and therefore it didn't succeed for the first time. Currently, the script returns foo first and bar second, while it should IMHO be the other way.

Try cli/copr_cli/package_build_order.py copr frostyx -p test-package-build-order-script

rebased onto 5aedd57863237f23c294cbc7f4fd853876349695

4 years ago

Yeah, you're right. Thanks for the review.
It should be fixed now.

We discussed that we should filter by tag by default, and we can keep username as optional filter.

rebased onto 0e4f7701cad807cc5650807660b6a39b00ebc001

4 years ago

I've rewritten it so that it's possible to filter the results in Koji either by tag or by username (or both).

PTAL.

Koji doesn't run createrepo after each build, so we should somehow
calculate with that (in future). In future we also should somehow detect
what packages in what versions were available at the beginning of project
iteration in copr (these points can wait)

The cli API has wrong design. It should IMO look like:
$ package-build-order [generic options] COMMAND [command specific options]
... where command is either copr or koji (and --tag needs to be
bound to koji only, id doesn't make sense with copr).
We could treat this PR as experimental thing, so this can wait as well.

But to make the script useful we need to at least output the NVRs, not
just name.

Running ./cli/copr_cli/package_build_order.py --tag f31 koji took ~2 minutes,
which is fine. But weird thing is that each package is mentioned only once.
What tags should we use to see the iteration?

The copr project should be specified by owner/project tuple, while owner
can be @group or user. You can keep two options, but it isn't that
convenient ... and the option name should be --ownername. This could
wait for now, but still.

For copr, you should probably allow filtering by chroot, somehow, and
work with BuildChroot, not Build. Otherwise we search through large
amount of builds and many of them may be irrelevant.

Btw., this failed after 50s,

$ time ./cli/copr_cli/package_build_order.py --username=iucar --projectname cran copr | tee /tmp/iucar
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/copr/v3/requests.py", line 101, in handle_errors
    response_json = response.json()
  File "/usr/lib/python3.7/site-packages/requests/models.py", line 897, in json
    return complexjson.loads(self.text, **kwargs)
  File "/usr/lib64/python3.7/site-packages/simplejson/__init__.py", line 518, in loads
    return _default_decoder.decode(s)
  File "/usr/lib64/python3.7/site-packages/simplejson/decoder.py", line 370, in decode
    obj, end = self.raw_decode(s)
  File "/usr/lib64/python3.7/site-packages/simplejson/decoder.py", line 400, in raw_decode
    return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "./cli/copr_cli/package_build_order.py", line 111, in <module>
    package_order_from_copr(args.config, args.username, args.projectname, args.list_each)
  File "./cli/copr_cli/package_build_order.py", line 28, in package_order_from_copr
    build_list = build_proxy.get_list(username, projectname)
  File "/usr/lib/python3.7/site-packages/copr/v3/helpers.py", line 59, in wrapper
    result = func(*args, **kwargs)
  File "/usr/lib/python3.7/site-packages/copr/v3/proxies/build.py", line 67, in get_list
    response = request.send()
  File "/usr/lib/python3.7/site-packages/copr/v3/requests.py", line 55, in send
    handle_errors(response)
  File "/usr/lib/python3.7/site-packages/copr/v3/requests.py", line 115, in handle_errors
    response=response)
copr.v3.exceptions.CoprRequestException: Request is not in JSON format, there is probably a bug in the API code.

real    0m50.151s
user    0m0.276s
sys     0m0.036s

rebased onto 26db43b49d620b17a5bb09a62927ed69088062b7

4 years ago

I've changed the API design as per your suggestion and the script now outputs the NVRs. In addition, for copr, I merged username and projectname into one argument -p owner/project.

But weird thing is that each package is mentioned only once.
What tags should we use to see the iteration?

I'm not sure what you mean by this comment. However, the problem might be fixed now that the NVRs are added?

rebased onto 02fbca3921e997f51e7e06506931064a9ca7b422

4 years ago

$ time ./cli/copr_cli/package_build_order.py --username=iucar --projectname cran copr | tee /tmp/iucar

This query doesn't fail for me, although it does take 4 minutes to process. I've changed the code to catch the exception.

rebased onto 95c1a978195c2b639a7e8a645aaa542b9359a45c

4 years ago

I've added an option to filter Copr results by chroot. However, I do it after querying frontend, so it wouldn't help for the larger projects. To filter the query itself, I'd probably need to change frontend's API, which I'd rather do in another PR.

Can you please take another look?

Traceback (most recent call last):
  File "./cli/copr_cli/package_build_order.py", line 129, in <module>
    args.func(args)
AttributeError: 'Namespace' object has no attribute 'func'

But weird thing is that each package is mentioned only once.
What tags should we use to see the iteration?

I mean, you usually need to bootstrap - you never build everything
from scratch by one iteration. Many packages should be printed
several times, with different NVRs.

rebased onto cbdd6c899a3de27b9f1ba52383264aeea1db764a

4 years ago

Fixed the traceback.

I mean, you usually need to bootstrap - you never build everything
from scratch by one iteration. Many packages should be printed
several times, with different NVRs.

This should be fixed now as well, as I'e added NVRs.

You need to specify either username or tag to list build order in Koji.
Traceback (most recent call last):
  File "./cli/copr_cli/package_build_order.py", line 130, in <module>
    args.func(args)
  File "./cli/copr_cli/package_build_order.py", line 96, in package_order_from_koji
    for build in sorted(builds, key=lambda i: i["completion_time"]):
UnboundLocalError: local variable 'builds' referenced before assignment

rebased onto f92ad69376d40fe26a7712c0c4541723a9d93833

4 years ago

Thanks, koji seems to work. The copr seems to work. API looks clean.

I'm testing ./cli/copr_cli/package_build_order.py copr -p iucar/cran --chroot fedora-rawhide-x86_64 now ...

Last thing; can we name the script like package-build-order? Having .py in %_bindir
is not nice, same as underscores...

That iucar/cran request did not work; perhaps API problem for large projects.
But:

$ ./cli/copr_cli/package_build_order.py copr -p @copr/copr-dev --chroot fedora-rawhide-x86_64
...
copr-frontend-1.164-1.git.16.9d103ad.fc31
copr-frontend-1.164-1.git.6045.9f869bc.fc31

E.g. copr-frontend-1.164-1.git.6045.9f869bc.fc31 is weird, I can not find it in
https://copr.fedorainfracloud.org/coprs/g/copr/copr-dev/package/copr-frontend/

It's a PR build (seems to me that the all builds tab doesn't show PR builds?): https://copr.fedorainfracloud.org/coprs/g/copr/copr-dev/build/1298902/

rebased onto 7443d9c42596a0d2f6ac4699ab5aad46b3534737

4 years ago

Last thing; can we name the script like package-build-order?

Done.

Ok, so PR builds should be filtered out if you ask for owner/project, and
should be covered if you ask for owner/project:pr:<ID>.

rebased onto 30c1334069f3c36dd712f1d21a3003efe3f2c618

4 years ago

Changed to filter out PR builds.

Looks much better, but

$ ./cli/copr_cli/package_build_order.py copr -p @copr/copr --chroot fedora-rawhide-x86_64
...
copr-rpmbuild-0.35-1.fc31
copr-frontend-1.162-1.fc31
copr-frontend-1.163-1.fc31
copr-rpmbuild-0.37-1.fc31

I miss copr-frontend-1.164-1 which succeeded in rawhide.

rebased onto 63f19adbe4f5198e0977d115dad752d3207d2980

4 years ago

Done. However, for this to work, we need commit eb48665 on frontend, as it provides build_chroot status over API.

I've added a patch to ansible so the script should now work on production.

Nice, I tried:
time ./cli/copr_cli/package_build_order.py copr -p @python/python3.9 --chroot fedora-rawhide-x86_64
and it took 2m38.563s, but it worked as I'd expect.

+1 thanks! Can be merged, but it's not 24h+.

I could release new version as well, but this still needs fix for PyPI (at least
it won't be installed by pip install copr).

The --list-each is still not ideal; you probably want to skip per package name, not per NVR.

Or maybe not, there could be two options for two separate tasks. But the --list-all really deserves proper argparse doc; what it does precisely.

rebased onto ab13090

4 years ago

Please fix the build, otherwise +1, I'll wrap the release and submit new bodhi updates.

Commit 6851f23 fixes this pull-request

Pull-Request has been merged by praiskup

4 years ago

Ok, we eventually merged ab13090 variant. PyPI release will be without that script.