#1953 API/CLI for last package builds
Merged a month ago by praiskup. Opened a month ago by praiskup.
copr/ praiskup/copr api-monitor-page  into  main

@@ -0,0 +1,31 @@ 

+ """

+ Pylintrc initialization methods.

+ """

+ 

+ import os

+ import sys

+ import subprocess

+ 

+ def init():

+     """

+     The main method, called in "init-hook=" config.

+     """

+ 

+     gitrootdir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()

+ 

+     # All modules depend, or can depend on python-common and python-copr

+     sys.path.insert(0, os.path.join(gitrootdir, 'common'))

+     sys.path.insert(0, os.path.join(gitrootdir, 'python'))

+     gitsubdir = subprocess.check_output(["git", "rev-parse", "--show-prefix"]).decode("utf-8").strip()

+ 

+     # Those sub-directories have the "nice" pattern, so setting the pythonpath

+     # here is trivial.

+     for nice_subdir in ["backend", "dist-git", "rpmbuild"]:

+         if gitsubdir.startswith(nice_subdir):

+             sys.path.insert(0, os.path.join(gitrootdir, nice_subdir))

+ 

+     # Those still need a special handling (and in future file movements).

+     if gitsubdir.startswith("frontend"):

+         sys.path.insert(0, os.path.join(gitrootdir, "frontend", "coprs_frontend"))

+     if gitsubdir.startswith("keygen"):

+         sys.path.insert(0, os.path.join(gitrootdir, "keygen", "src"))

@@ -0,0 +1,21 @@ 

+ """

+ Helpers for the 'copr-cli' command.

+ """

+ 

+ output_format_help = """

+ Set the formatting style. We recommend using json, which prints the required data

+ in json format. The text format prints the required data in a column, one piece of

+ information per line. The text-row format prints all information separated

+ by a space on a single line.

+ """

+ 

+ def cli_use_output_format(parser, default='json'):

+     """

+     Add '--output-format' option to given parser.

+     """

+     parser.add_argument(

+         "--output-format",

+         choices=["text", "json", "text-row"],

+         help=output_format_help,

+         default=default,

+     )

file modified
+26 -141
@@ -28,7 +28,10 @@ 

      CoprNoConfigException, CoprConfigException, CoprNoResultException,

  )

  from copr.v3.pagination import next_page

- from .util import ProgressBar, json_dumps, serializable

+ from copr_cli.helpers import cli_use_output_format

+ from copr_cli.monitor import cli_monitor_parser

+ from copr_cli.printers import cli_get_output_printer as get_printer

+ from .util import ProgressBar, serializable

  from .build_config import MockProfile

  

  
@@ -73,13 +76,6 @@ 

  

  """

  

- output_format_help = """

- Set the formatting style. We recommend using json, which prints the required data

- in json format. The text format prints the required data in a column, one piece of

- information per line. The text-row format prints all information separated

- by a space on a single line.

- """

- 

  try:

      input = raw_input

  except NameError:
@@ -131,128 +127,6 @@ 

      return buildopts

  

  

- class AbstractPrinter(object):

-     """Abstract class defining mandatory methods of printer classes"""

- 

-     def __init__(self, fields):

-         """Represents the data we want to print.

-         Supports callable lambda function which takes exactly one argument"""

-         self.fields = fields

- 

-     def add_data(self, data):

-         """Initialize the data to be printed"""

-         raise NotImplementedError

- 

-     def finish(self):

-         """Print the data according to the set format"""

-         raise NotImplementedError

- 

- 

- class RowTextPrinter(AbstractPrinter):

-     """The class takes care of printing the data in row text format"""

- 

-     def finish(self):

-         pass

- 

-     def add_data(self, data):

-         row_data = []

-         for field in self.fields:

-             if callable(field):

-                 row_data.append(str(field(data)))

-             else:

-                 row_data.append(str(data[field]))

-         print("\t".join(row_data))

- 

- 

- class ColumnTextPrinter(AbstractPrinter):

-     """The class takes care of printing the data in column text format"""

- 

-     first_line = True

- 

-     def finish(self):

-         pass

- 

-     def add_data(self, data):

-         if not self.first_line:

-             print()

-         self.first_line = False

-         for field in self.fields:

-             if callable(field):

-                 print("{0}: {1}".format(field.__code__.co_varnames[0], str(field(data))))

-             else:

-                 print("{0}: {1}".format(field, str(data[field])))

- 

- 

- class JsonPrinter(AbstractPrinter):

-     """The class takes care of printing the data in json format"""

- 

-     def _get_result_json(self, data):

-         result = {}

-         for field in self.fields:

-             if callable(field):

-                 name = field.__code__.co_varnames[0]

-                 result[name] = field(data)

-             else:

-                 result[field] = data[field]

-         return json_dumps(result)

- 

-     def add_data(self, data):

-         print(self._get_result_json(data))

- 

-     def finish(self):

-         pass

- 

- 

- class JsonPrinterListCommand(JsonPrinter):

-     """

-     The class takes care of printing the data in list in json format

- 

-     For performance reasons we cannot simply add everything to a list and then

-     use `json_dumps` function to print all JSON at once. For large projects this

-     would mean minutes of no output, which is not user-friendly.

- 

-     The approach here is to utilize `json_dumps` to convert each object to

-     a JSON string and print it immediately. We then manually take care of

-     opening and closing list brackets and separators.

-     """

- 

-     first_line = True

- 

-     def add_data(self, data):

-         self.start()

-         result = self._get_result_json(data)

-         for line in result.split("\n"):

-             print("    {0}".format(line))

- 

-     def start(self):

-         """

-         This is called before printing the first object

-         """

-         if self.first_line:

-             self.first_line = False

-             print("[")

-         else:

-             print("    ,")

- 

-     def finish(self):

-         """

-         This is called by the user after printing all objects

-         """

-         if not self.first_line:

-             print("]")

- 

- 

- def get_printer(output_format, fields, list_command=False):

-     """According to output_format decide which object of printer to return"""

-     if output_format == "json":

-         if list_command:

-             return JsonPrinterListCommand(fields)

-         return JsonPrinter(fields)

-     if output_format == "text":

-         return ColumnTextPrinter(fields)

-     return RowTextPrinter(fields)

- 

- 

  class Commands(object):

      def __init__(self, config_path):

          self.config_path = config_path or '~/.config/copr'
@@ -312,6 +186,14 @@ 

              owner = self.config["username"]

          return owner, name

  

+     def build_url(self, build_id):

+         """

+         Return the "generic" predictable url for build_id, which redirects

+         to the final owner/project/build_id route.

+         """

+         return urljoin(self.config["copr_url"],

+                        "/coprs/build/{0}".format(build_id))

+ 

      def parse_chroot_path(self, path):

          """

          Take a `path` in an `owner/project/chroot` format and return a tuple of
@@ -524,8 +406,7 @@ 

              print("Build was added to {0}:".format(builds[0].projectname))

  

              for build in builds:

-                 url = urljoin(self.config["copr_url"], "/coprs/build/{0}".format(build.id))

-                 print("  {0}".format(url))

+                 print("  {0}".format(self.build_url(build.id)))

  

              build_ids = [build.id for build in builds]

              print("Created builds: {0}".format(" ".join(map(str, build_ids))))
@@ -1247,8 +1128,7 @@ 

      parser_builds.add_argument("project", help="Which project's builds should be listed.\

                                 Can be just a name of the project or even in format\

                                 username/project or @groupname/project.")

-     parser_builds.add_argument("--output-format", choices=["text", "json", "text-row"],

-                                help=output_format_help)

+     cli_use_output_format(parser_builds, default=None)

      parser_builds.set_defaults(func="action_list_builds")

  

      #########################################################
@@ -1514,8 +1394,7 @@ 

  

      parser_get_chroot = subparsers.add_parser("get-chroot", help="Get chroot of a project")

      parser_get_chroot.add_argument("coprchroot", help="Path to a project chroot as owner/project/chroot or project/chroot")

-     parser_get_chroot.add_argument("--output-format", default="json", choices=["text", "json", "text-row"],

-                                    help=output_format_help)

+     cli_use_output_format(parser_get_chroot)

      parser_get_chroot.set_defaults(func="action_get_chroot")

  

      parser_list_chroots = subparsers.add_parser("list-chroots", help="List all currently available chroots.")
@@ -1639,8 +1518,7 @@ 

                                        help="Also display data related to the latest succeeded build for the package.")

      parser_list_packages.add_argument("--with-all-builds", action="store_true",

                                        help="Also display data related to the builds for the package.")

-     parser_list_packages.add_argument("--output-format", default="json", choices=["text", "json", "text-row"],

-                                       help=output_format_help)

+     cli_use_output_format(parser_list_packages)

      parser_list_packages.set_defaults(func="action_list_packages")

  

      # package names listing
@@ -1664,8 +1542,7 @@ 

                                      help="Also display data related to the latest succeeded build for each package.")

      parser_get_package.add_argument("--with-all-builds", action="store_true",

                                      help="Also display data related to the builds for each package.")

-     parser_get_package.add_argument("--output-format", default="json", choices=["text", "json", "text-row"],

-                                     help=output_format_help)

+     cli_use_output_format(parser_get_package)

      parser_get_package.set_defaults(func="action_get_package")

  

      # package deletion
@@ -1795,6 +1672,9 @@ 

              help=request_help.format('builder'))

      parser_permissions_request.set_defaults(func='action_permissions_request')

  

+     # package monitoring

+     cli_monitor_parser(subparsers)

+ 

      if argcomplete:

          argcomplete.autocomplete(parser)

      return parser
@@ -1832,7 +1712,12 @@ 

              return

  

          commands = Commands(arg.config)

-         getattr(commands, arg.func)(arg)

+         if isinstance(arg.func, str):

+             # Call self method by its name

+             getattr(commands, arg.func)(arg)

+         else:

+             # Call external command method

+             arg.func(commands, arg)

  

      except KeyboardInterrupt:

          sys.stderr.write("\nInterrupted by user.")

@@ -0,0 +1,109 @@ 

+ """

+ The 'copr-cli monitor' implementation

+ """

+ 

+ from argparse import ArgumentTypeError

+ 

+ from copr_cli.helpers import cli_use_output_format

+ from copr_cli.printers import cli_get_output_printer

+ 

+ 

+ DEFAULT_FIELDS = [

+     "name",

+     "chroot",

+     "build_id",

+     "state",

+ ]

+ 

+ ADDITIONAL_FIELDS = [

+     "url_build_log",

+     "url_backend_log",

+ ]

+ 

+ ALLOWED_FIELDS = DEFAULT_FIELDS + ADDITIONAL_FIELDS + [

+     "url_build"

+ ]

+ 

+ 

+ def cli_monitor_action(commands, args):

+     """

+     Get info about the latest chroot builds for requested packages.

+     """

+     ownername, projectname = commands.parse_name(args.project)

+ 

+     fields = None

+     if args.fields:

+         fields = [arg.strip() for arg in args.fields.split(",")]

+ 

+         bad_fields = []

+         for field in fields:

+             if field in ALLOWED_FIELDS:

+                 continue

+             bad_fields.append(field)

+ 

+         if bad_fields:

+             raise ArgumentTypeError(

+                 "Unknown field(s) specified in --fields: " +

+                 str(bad_fields) + ", " +

+                 "allowed: " + str(ALLOWED_FIELDS),

+             )

+     else:

+         fields = DEFAULT_FIELDS

+ 

+     # Package name, and chroot name are automatically in the output.

+     requested_fields = list(set(fields).intersection(ADDITIONAL_FIELDS)) or None

+ 

+     data = commands.client.monitor_proxy.monitor(

+         ownername=ownername, projectname=projectname,

+         additional_fields=requested_fields,

+     )

+ 

+ 

+     printer = cli_get_output_printer(args.output_format, fields, True)

+     for package in data["packages"]:

+         for chroot_name, chroot in package["chroots"].items():

+             data = {

+                 "name": package["name"],

+                 "chroot": chroot_name,

+             }

+             if "url_build" in fields:

+                 data["url_build"] = commands.build_url(chroot["build_id"])

+             data.update(chroot)

+             data = {

+                 key: value

+                 for key, value in data.items()

+                 if key in fields

+             }

+             printer.add_data(data)

+     printer.finish()

+ 

+ 

+ def cli_monitor_parser(subparsers):

+     """

+     Append "copr-cli monitor" sub-parser.

+     """

+     parser_monitor = subparsers.add_parser(

+             "monitor",

+             help="Monitor package build state",

+     )

+     parser_monitor.set_defaults(func=cli_monitor_action)

+     parser_monitor.add_argument(

+         "project", help=("Which project's packages should be listed. "

+                          "Can be just a name of the project or even in format "

+                          "username/project or @groupname/project."))

+     parser_monitor.add_argument(

+             "--dirname",

+             help=("project (sub)directory name, e.g. 'foo:pr:125', "

+                   "by default just 'foo' is used"))

+ 

+     cli_use_output_format(parser_monitor)

+ 

+     parser_monitor.add_argument(

+         "--fields",

+         help=(

+             "A comma-separated list (ordered) of fields to be printed. "

+             "Possible values: {0}.  Note that url_build* options might "

+             "significantly prolong the server response time.".format(

+                ", ".join(ALLOWED_FIELDS))

+         ),

+     )

@@ -0,0 +1,127 @@ 

+ """

+ Abstraction for printing data to copr-cli's stdout

+ """

+ 

+ from .util import json_dumps

+ 

+ 

+ class AbstractPrinter(object):

+     """Abstract class defining mandatory methods of printer classes"""

+ 

+     def __init__(self, fields):

+         """Represents the data we want to print.

+         Supports callable lambda function which takes exactly one argument"""

+         self.fields = fields

+ 

+     def add_data(self, data):

+         """Initialize the data to be printed"""

+         raise NotImplementedError

+ 

+     def finish(self):

+         """Print the data according to the set format"""

+         raise NotImplementedError

+ 

+ 

+ class RowTextPrinter(AbstractPrinter):

+     """The class takes care of printing the data in row text format"""

+ 

+     def finish(self):

+         pass

+ 

+     def add_data(self, data):

+         row_data = []

+         for field in self.fields:

+             if callable(field):

+                 row_data.append(str(field(data)))

+             else:

+                 row_data.append(str(data[field]))

+         print("\t".join(row_data))

+ 

+ 

+ class ColumnTextPrinter(AbstractPrinter):

+     """The class takes care of printing the data in column text format"""

+ 

+     first_line = True

+ 

+     def finish(self):

+         pass

+ 

+     def add_data(self, data):

+         if not self.first_line:

+             print()

+         self.first_line = False

+         for field in self.fields:

+             if callable(field):

+                 print("{0}: {1}".format(field.__code__.co_varnames[0], str(field(data))))

+             else:

+                 print("{0}: {1}".format(field, str(data[field])))

+ 

+ 

+ class JsonPrinter(AbstractPrinter):

+     """The class takes care of printing the data in json format"""

+ 

+     def _get_result_json(self, data):

+         result = {}

+         for field in self.fields:

+             if callable(field):

+                 name = field.__code__.co_varnames[0]

+                 result[name] = field(data)

+             else:

+                 result[field] = data[field]

+         return json_dumps(result)

+ 

+     def add_data(self, data):

+         print(self._get_result_json(data))

+ 

+     def finish(self):

+         pass

+ 

+ 

+ class JsonPrinterListCommand(JsonPrinter):

+     """

+     The class takes care of printing the data in list in json format

+ 

+     For performance reasons we cannot simply add everything to a list and then

+     use `json_dumps` function to print all JSON at once. For large projects this

+     would mean minutes of no output, which is not user-friendly.

+ 

+     The approach here is to utilize `json_dumps` to convert each object to

+     a JSON string and print it immediately. We then manually take care of

+     opening and closing list brackets and separators.

+     """

+ 

+     first_line = True

+ 

+     def add_data(self, data):

+         self.start()

+         result = self._get_result_json(data)

+         for line in result.split("\n"):

+             print("    {0}".format(line))

+ 

+     def start(self):

+         """

+         This is called before printing the first object

+         """

+         if self.first_line:

+             self.first_line = False

+             print("[")

+         else:

+             print("    ,")

+ 

+     def finish(self):

+         """

+         This is called by the user after printing all objects

+         """

+         if not self.first_line:

+             print("]")

+ 

+ 

+ def cli_get_output_printer(output_format, fields, list_command=False):

+     """According to output_format decide which object of printer to return"""

+     if output_format == "json":

+         if list_command:

+             return JsonPrinterListCommand(fields)

+         return JsonPrinter(fields)

+     if output_format == "text":

+         return ColumnTextPrinter(fields)

+     return RowTextPrinter(fields)

@@ -32,8 +32,6 @@ 

  # where to send notice about raised legal flag

  #SEND_LEGAL_TO = ['root@localhost', 'somebody@somewhere.com']

  

- # DEBUG = False

- DEBUG = True

  SQLALCHEMY_ECHO = False

  

  #CSRF_ENABLED = True
@@ -136,7 +134,31 @@ 

  MIN_BUILD_TIMEOUT = 0

  MAX_BUILD_TIMEOUT = 108000

  

+ 

+ #############################

+ ##### DEBUGGING Section #####

+ 

+ # The following options are not designed (or even safe) to be used in production

+ # systems!

+ 

+ # Turn on the Flask debugger (and some Copr internal mechanisms):

+ # https://flask.palletsprojects.com/en/latest/debugging/

+ DEBUG = False

+ 

+ # Enable flask-profiler:

+ # https://pypi.org/project/flask_profiler/

+ # Setting this to True requires the flask_profiler installed

+ PROFILER = False

+ 

  # Provide special /ma/diff*.txt routes for memory management analysis.  This

  # should never be enabled in production (see the memory_analyzer.py module for

  # more info).

- #MEMORY_ANALYZER = False

+ MEMORY_ANALYZER = False

+ 

+ # Turn-on the in-code checkpoints (additional logging info output) which measure

+ # time since the beginning of the request, and between each checkpoints.  See

+ # the 'measure.py' module for more info.

+ DEBUG_CHECKPOINTS = False

+ 

+ ##### DEBUGGING Section #####

+ #############################

@@ -101,9 +101,11 @@ 

  from coprs.views import api_ns

  from coprs.views.api_ns import api_general

  from coprs.views import apiv3_ns

- from coprs.views.apiv3_ns import (apiv3_general, apiv3_builds, apiv3_packages, apiv3_projects, apiv3_project_chroots,

-                                   apiv3_modules, apiv3_build_chroots, apiv3_mock_chroots,

-                                   apiv3_permissions, apiv3_webhooks)

+ from coprs.views.apiv3_ns import (

+     apiv3_general, apiv3_builds, apiv3_packages, apiv3_projects,

+     apiv3_project_chroots, apiv3_modules, apiv3_build_chroots,

+     apiv3_mock_chroots, apiv3_permissions, apiv3_webhooks, apiv3_monitor,

+ )

  

  from coprs.views import batches_ns

  from coprs.views.batches_ns import coprs_batches

@@ -140,6 +140,10 @@ 

      # Default value for temporary projects

      DELETE_AFTER_DAYS = 60

  

+     # Turn-on the in-code checkpoints (additional logging info output).  See the

+     # 'measure.py' module for more info.

+     DEBUG_CHECKPOINTS = False

+ 

  

  class ProductionConfig(Config):

      DEBUG = False

@@ -47,7 +47,9 @@ 

          """

          Return status code for a given exception

          """

-         return getattr(error, "code", 500)

+         code = getattr(error, "code", 500)

+         return code if code is not None else 500

+ 

  

      def message(self, error):  # pylint: disable=no-self-use

          """

@@ -8,7 +8,7 @@ 

  

  from sqlalchemy.sql import text

  from sqlalchemy.sql.expression import not_

- from sqlalchemy.orm import joinedload, selectinload, load_only

+ from sqlalchemy.orm import joinedload, selectinload, load_only, contains_eager

  from sqlalchemy import func, desc, or_, and_

  from sqlalchemy.sql import false,true

  from werkzeug.utils import secure_filename
@@ -40,6 +40,7 @@ 

  from coprs.logic.coprs_logic import MockChrootsLogic

  from coprs.logic.packages_logic import PackagesLogic

  from coprs.logic.batches_logic import BatchesLogic

+ from coprs.measure import checkpoint

  

  from .helpers import get_graph_parameters

  log = app.logger
@@ -393,7 +394,7 @@ 

      def get_copr_builds_list(cls, copr, dirname=None):

          query = models.Build.query.filter(models.Build.copr_id==copr.id)

          if dirname:

-             copr_dir = coprs_logic.CoprDirsLogic.get_by_copr(copr, dirname).one()

+             copr_dir = coprs_logic.CoprDirsLogic.get_by_copr(copr, dirname)

              query = query.filter(models.Build.copr_dir_id==copr_dir.id)

          query = query.options(selectinload('build_chroots'), selectinload('package'))

          return query
@@ -689,15 +690,52 @@ 

          return batch

  

      @classmethod

+     def assign_buildchroots(cls, build, chroot_names, status=None,

+                             git_hashes=None, **kwargs):

+         """

+         For each chroot_name allocate a new BuildChroot instance and assign it

+         to the build.

+         """

+         something_added = False

+         for chroot in chroot_names:

+             something_added = True

+             additional_args = {}

+             if git_hashes:

+                 additional_args["git_hash"] = git_hashes.get(chroot)

+             if status is None:

+                 status = StatusEnum("waiting")

+             buildchroot = BuildChrootsLogic.new(

+                 build=build,

+                 mock_chroot=chroot,

+                 status=status,

+                 **kwargs,

+                 **additional_args,

+             )

+             db.session.add(buildchroot)

+         return something_added

+ 

+     @classmethod

+     def assign_buildchroots_from_package(cls, build, **kwargs):

+         """

+         If build has already assigned build.package, create and assign new set

+         of BuildChroot instances according to 'build.package.chroots'.  This

+         e.g. respects the Copr.chroot_denylist config.

+         """

+         if not build.package:

+             return False

+         return cls.assign_buildchroots(

+             build,

+             build.package.chroots,

+             **kwargs,

+         )

+ 

+     @classmethod

      def add(cls, user, pkgs, copr, source_type=None, source_json=None,

              repos=None, chroots=None, timeout=None, enable_net=True,

              git_hashes=None, skip_import=False, background=False, batch=None,

              srpm_url=None, copr_dirname=None, bootstrap=None, isolation=None,

              package=None, after_build_id=None, with_build_id=None):

  

-         if chroots is None:

-             chroots = []

- 

          coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(

              copr, "Can't build while there is an operation in progress: {action}")

          users_logic.UsersLogic.raise_if_cant_build_in_copr(
@@ -754,21 +792,18 @@ 

          if timeout:

              build.timeout = timeout or app.config["DEFAULT_BUILD_TIMEOUT"]

  

-         db.session.add(build)

- 

-         for chroot in chroots:

-             # Chroots were explicitly set per-build.

-             git_hash = None

-             if git_hashes:

-                 git_hash = git_hashes.get(chroot.name)

-             buildchroot = BuildChrootsLogic.new(

-                 build=build,

-                 status=chroot_status,

-                 mock_chroot=chroot,

-                 git_hash=git_hash,

-             )

-             db.session.add(buildchroot)

+         if not chroots and package:

+             chroots = package.chroots

+         if not chroots:

+             chroots = []

  

+         db.session.add(build)

+         cls.assign_buildchroots(

+             build,

+             chroots,

+             git_hashes=git_hashes,

+             status=chroot_status,

+         )

          return build

  

      @classmethod
@@ -813,17 +848,7 @@ 

              submitted_by=submitted_by,

          )

          db.session.add(build)

- 

-         status = StatusEnum("waiting")

-         for chroot in package.chroots:

-             buildchroot = BuildChrootsLogic.new(

-                 build=build,

-                 status=status,

-                 mock_chroot=chroot,

-                 git_hash=None

-             )

-             db.session.add(buildchroot)

- 

+         cls.assign_buildchroots_from_package(build)

          cls.process_update_callback(build)

          return build

  
@@ -917,19 +942,10 @@ 

                  new_status = StatusEnum("importing")

                  chroot_status=StatusEnum("waiting")

                  if not build.build_chroots:

-                     # create the BuildChroots from Package setting, if not

-                     # already set explicitly for concrete build

-                     added = False

-                     for chroot in build.package.chroots:

-                         added = True

-                         buildchroot = BuildChrootsLogic.new(

-                             build=build,

-                             status=chroot_status,

-                             mock_chroot=chroot,

-                             git_hash=None,

-                         )

-                         db.session.add(buildchroot)

-                     if not added:

+                     # When BuildChroots are not yet allocated, allocate them now

+                     # from the assigned Package setting.

+                     if not cls.assign_buildchroots_from_package(

+                             build, status=chroot_status, git_hash=None):

                          new_status = StatusEnum("failed")

                  else:

                      for buildchroot in build.build_chroots:
@@ -1400,6 +1416,141 @@ 

  

  class BuildsMonitorLogic(object):

      @classmethod

+     def package_build_chroots_query(cls, copr_dir, mock_chroot_ids):

+         """

+         Return an SQL query returning all BuildChroots assigned to given CoprDir

+         (copr_dir) and MockChroot's (mock_chroot_ids).  The output is sorted by

+         Package.name, and then by Build.id (so callers can easily skip the

+         uninteresting entries).

+         """

+         return (

+             models.BuildChroot.query

+             .join(models.Build)

+             .join(models.Package)

+             .options(

+                 load_only("build_id", "status", "mock_chroot_id", "result_dir"),

+                 contains_eager("build").load_only("package_id", "copr_dir_id")

+                     .contains_eager("package").load_only("name"),

+             )

+             .filter(models.BuildChroot.mock_chroot_id.in_(mock_chroot_ids))

+             .filter(models.Build.copr_dir==copr_dir)

+             .order_by(

+                 models.Package.name.asc(),

+                 models.Build.id.desc(),

+             )

+         )

+ 

+     @classmethod

+     def package_build_chroots(cls, copr_dir):

+         """

+         For each Packge in CoprDir (copr_dir) and its assigned and enabled

+         CoprChroots, return a list of the latest BuildChroots, if any.

+         The output format is:

+             [{

+                 "name": "PackageName",

+                 "chroots": [ <BuildChroot>, <BuildChroot>, ...  ],

+             }, ...]

+         """

+ 

+         class _Finder:

+             def __init__(self, find_mock_chroot_ids):

+                 self.searching_for = set(find_mock_chroot_ids)

+                 self.package_name = None

+                 self._reset()

+ 

+             def _reset(self):

+                 self.mock_chroot_ids_found = set()

+                 self.package_name = None

+                 self.chroots = []

+ 

+             def get_data(self):

+                 """

+                 Return the dictionary to be yielded, or None if no data are

+                 already processed.

+                 """

+                 if not self.chroots:

+                     # no package in the result set

+                     return None

+                 return {

+                     "name": self.package_name,

+                     "chroots": self.chroots,

+                 }

+ 

+             def _add_item(self, build_chroot):

+                 self.mock_chroot_ids_found.add(build_chroot.mock_chroot_id)

+                 self.chroots.append(build_chroot)

+                 self.package_name = build_chroot.build.package.name

+ 

+             def add(self, build_chroot):

+                 """

+                 Process one BuildChroot instance.  Return the get_data() output

+                 if we should flush the package out to caller, or None.

+                 """

+                 package_name = build_chroot.build.package.name

+ 

+                 if self.package_name is None:

+                     # first package

+                     self._add_item(build_chroot)

+                     return None

+ 

+                 if package_name == self.package_name:

+                     # additional chroot for the same package

+                     mock_chroot_id = build_chroot.mock_chroot_id

+                     if mock_chroot_id in self.mock_chroot_ids_found:

+                         # we already have status for this

+                         return None

+                     self._add_item(build_chroot)

+                     return None

+ 

+                 # a different package, flush the old one

+                 return_value = self.get_data()

+                 self._reset()

+                 self._add_item(build_chroot)

+                 return return_value

+ 

+         mock_chroot_ids = {mch.id for mch in copr_dir.copr.active_chroots}

+         package_data = _Finder(mock_chroot_ids)

+         for bch in cls.package_build_chroots_query(copr_dir, mock_chroot_ids).yield_per(1000):

+             if value := package_data.add(bch):

+                 yield value

+ 

+         if value := package_data.get_data():

+             yield value

+ 

+ 

+     @classmethod

+     def last_buildchroots(cls, pkg_ids, mock_chroot_ids):

+         """

+         Query the BuildChroot for given list of package IDs, and mock chroot IDs

+         """

+         builds_ids = (

+             models.Build.query.join(models.BuildChroot)

+                   .filter(models.Build.package_id.in_(pkg_ids))

+                   .filter(models.BuildChroot.mock_chroot_id.in_(mock_chroot_ids))

+                   .with_entities(

+                       models.Build.package_id.label('package_id'),

+                       func.max(models.Build.id).label('build_id').label("build_id"),

+                       models.BuildChroot.mock_chroot_id,

+                   )

+                   .group_by(

+                       models.Build.package_id,

+                       models.BuildChroot.mock_chroot_id,

+                   )

+                   .subquery()

+         )

+ 

+         return (models.BuildChroot.query

+             .join(

+                 builds_ids,

+                 and_(builds_ids.c.build_id == models.BuildChroot.build_id,

+                      builds_ids.c.mock_chroot_id == models.BuildChroot.mock_chroot_id))

+             .add_columns(

+                 builds_ids.c.package_id.label("package_id"),

+             )

+         )

+ 

+ 

+     @classmethod

      def get_monitor_data(cls, copr, per_page=50, page=1,

                           paginate_if_more_than=1000):

          """
@@ -1431,39 +1582,22 @@ 

          pkg_ids = [package.id for package in pagination.items]

          mock_chroot_ids = [mch.id for mch in copr.active_chroots]

  

-         builds_ids = (

-             models.Build.query.join(models.BuildChroot)

-                   .filter(models.Build.package_id.in_(pkg_ids))

-                   .filter(models.BuildChroot.mock_chroot_id.in_(mock_chroot_ids))

-                   .with_entities(

-                       models.Build.package_id.label('package_id'),

-                       func.max(models.Build.id).label('build_id').label("build_id"),

-                       models.BuildChroot.mock_chroot_id,

-                   )

-                   .group_by(

-                       models.Build.package_id,

-                       models.BuildChroot.mock_chroot_id,

-                   )

-                   .subquery()

-         )

- 

-         query = (models.BuildChroot.query

-             .join(

-                 builds_ids,

-                 and_(builds_ids.c.build_id == models.BuildChroot.build_id,

-                      builds_ids.c.mock_chroot_id == models.BuildChroot.mock_chroot_id))

-             .add_columns(

-                 builds_ids.c.package_id.label("package_id"),

-             )

-         )

+         query = BuildsMonitorLogic.last_buildchroots(pkg_ids, mock_chroot_ids)

  

          mapper = {}

          for package in pagination.items:

              mapper[package.id] = {}

  

+         checkpoint("large query start")

+         first = True

          for build_chroot, package_id in query:

+             if first:

+                 checkpoint("large query done")

+             first = False

              mapper[package_id][build_chroot.mock_chroot.name] = build_chroot

  

+         checkpoint("buildchroot => package mapped")

+ 

          for package in pagination.items:

              package.latest_build_chroots = []

              for chroot in copr.active_chroots_sorted:

@@ -543,8 +543,32 @@ 

  

  class CoprDirsLogic(object):

      @classmethod

+     def get_by_copr_safe(cls, copr, dirname):

+         """

+         Return _query_ for getting CoprDir by Copr and dirname

+         """

+         return (db.session.query(models.CoprDir)

+                 .join(models.Copr)

+                 .filter(models.Copr.id==copr.id)

+                 .filter(models.CoprDir.name==dirname)).first()

+ 

+     @classmethod

+     def get_by_copr(cls, copr, dirname):

+         """

+         Return CoprDir instance per given Copr instance and dirname.  Raise

+         ObjectNotFound if it doesn't exist.

+         """

+         coprdir = cls.get_by_copr_safe(copr, dirname)

+         if not coprdir:

+             raise exceptions.ObjectNotFound(

+                 "Dirname '{}' doesn't exist in '{}' copr".format(

+                     dirname,

+                     copr.full_name))

+         return coprdir

+ 

+     @classmethod

      def get_or_create(cls, copr, dirname, main=False):

-         copr_dir = cls.get_by_copr(copr, dirname).first()

+         copr_dir = cls.get_by_copr_safe(copr, dirname)

  

          if copr_dir:

              return copr_dir
@@ -557,12 +581,6 @@ 

          db.session.add(copr_dir)

          return copr_dir

  

-     @classmethod

-     def get_by_copr(cls, copr, dirname):

-         return (db.session.query(models.CoprDir)

-                 .join(models.Copr)

-                 .filter(models.Copr.id==copr.id)

-                 .filter(models.CoprDir.name==dirname))

  

      @classmethod

      def get_by_ownername(cls, ownername, dirname):

@@ -0,0 +1,54 @@ 

+ """

+ Helper methods for measuring flask performance.

+ """

+ 

+ import datetime

+ 

+ from coprs import app

+ 

+ CONTEXT = None

+ 

+ 

+ class CheckPointContext:

+     """

+     the checkpoint's CONTEXT variable

+     """

+     def __init__(self):

+         self.start = datetime.datetime.now()

+         self.last = self.start

+ 

+ 

+ def checkpoint_start(force=False):

+     """

+     Enable checkpoints by flask global variable.

+     """

+     global CONTEXT  # pylint: disable=global-statement

+     if force or app.config["DEBUG_CHECKPOINTS"]:

+         CONTEXT = CheckPointContext()

+ 

+ 

+ def checkpoint(message):

+     """

+     Print useful timing info for the CONTEXT, prefixed with MESSAGE.

+     You can enable this by `DEBUG_CHECKPOINTS = True` config option, or

+     by calling `checkpoint_start(force=True)`.  Usage:

+ 

+         checkpoint("start")

+         some_expensive_action()

+         checkpoint("Expensive action finished")

+ 

+     Then the stdout output is:

+ 

+         start                         : 0:00:00.148130 (full time 0:00:00.148130)

+         Expensive action finished     : 0:00:04.223232 (full time 0:00:04.371362)

+     """

+ 

+     if CONTEXT is None:

+         return

+     start = CONTEXT.start

+     last = CONTEXT.last

+     now = datetime.datetime.now()

+ 

+     app.logger.info("%30s: %s (full time %s)",

+                     message, now-last, now-start)

+     CONTEXT.last = now

@@ -252,3 +252,42 @@ 

          if field.name in formdata.keys():

              continue

          formdata[field.name] = field.default

+ 

+ 

+ def streamed_json_array_response(array_or_generator, message, field="data"):

+     """

+     Helper response to stream large JSON API arrays (ARRAY_OR_GENERATOR).  We

+     keep the layout of the output like::

+ 

+         {

+             "output": "ok",

+             "message: MESSAGE,

+             "<FIELD>": [ ITEM, ITEM, ...,]

+         }

+ 

+     .. as it is expected by clients.  We iterate continuously over the array

+     items (or fetch from generator), so we don't have to keep the large dataset

+     in memory (or wait till it is fully fetched from DB).

+     """

+ 

+     def _stream():

+         start_string = (

+             '{{'

+             '"output": "ok",'

+             '"message": {message},'

+             '{field}: [\n'

+         )

+         yield start_string.format(message=json.dumps(message),

+                                   field=json.dumps(field))

+         for item in array_or_generator:

+             if start_string:

+                 yield json.dumps(item)

+                 start_string = None

+             else:

+                 yield ",\n" + json.dumps(item)

+         yield "]}"

+ 

+     return flask.Response(

+         _stream(),

+         mimetype="application/json",

+     )

@@ -0,0 +1,100 @@ 

+ """

+ /api_3/monitor routes

+ """

+ 

+ import flask

+ 

+ from coprs.exceptions import BadRequest

+ from coprs.logic.builds_logic import BuildsMonitorLogic

+ from coprs.logic.coprs_logic import CoprDirsLogic

+ from coprs.views.apiv3_ns import (

+     apiv3_ns,

+     GET,

+     get_copr,

+     query_params,

+     streamed_json_array_response,

+ )

+ 

+ from coprs.measure import checkpoint

+ 

+ def monitor_generator(copr_dir, additional_fields):

+     """

+     Continuosly fill-up the package_monitor() buffer.

+     """

+     anti_garbage_collector = set([copr_dir])

+     packages = BuildsMonitorLogic.package_build_chroots(copr_dir)

+     first = True

+     for package in packages:

+         if first is True:

+             checkpoint("First package queried")

+             first = False

+         chroots = {}

+         for bch in package["chroots"]:

+             chroot = chroots[bch.name] = {}

+             for attr in ["state", "status", "build_id"]:

+                 chroot[attr] = getattr(bch, attr)

+             if "url_build_log" in additional_fields:

+                 chroot["url_build_log"] = bch.rpm_live_log_url

+             if "url_backend_log" in additional_fields:

+                 chroot["url_backend_log"] = bch.rpm_backend_log_url

+             # anti-gc, this is a very small set of items

+             anti_garbage_collector.add(bch.mock_chroot)

+             anti_garbage_collector.add(bch.build.copr_dir)

+         yield {

+             "name": package["name"],

+             "chroots": chroots,

+         }

+     checkpoint("Last package queried")

+ 

+ 

+ @apiv3_ns.route("/monitor", methods=GET)

+ @query_params()

+ def package_monitor(ownername, projectname, project_dirname=None):

+     """

+     For list of the project packages return list of JSON dictionaries informing

+     about status of the last chroot builds (status, build log, etc.).

+     """

+     checkpoint("API3 monitor start")

+ 

+     additional_fields = flask.request.args.getlist("additional_fields[]")

+ 

+     copr = get_copr(ownername, projectname)

+ 

+     valid_additional_fields = [

+         "url_build_log",

+         "url_backend_log",

+         "url_build",

+     ]

+ 

+     if additional_fields:

+         additional_fields = set(additional_fields)

+         bad_fields = []

+         for field in sorted(additional_fields):

+             if field not in valid_additional_fields:

+                 bad_fields += [field]

+         if bad_fields:

+             raise BadRequest(

+                 "Wrong additional_fields argument(s): " +

+                 ", ".join(bad_fields)

+             )

+     else:

+         additional_fields = set()

+ 

+     if project_dirname:

+         copr_dir = CoprDirsLogic.get_by_copr(copr, project_dirname)

+     else:

+         copr_dir = copr.main_dir

+ 

+     # Preload those to avoid the error sqlalchemy.orm.exc.DetachedInstanceError

+     # http://sqlalche.me/e/13/bhk3

+     _ = copr_dir.copr.active_chroots

+     _ = copr_dir.copr.group

+ 

+     try:

+         return streamed_json_array_response(

+             monitor_generator(copr_dir, additional_fields),

+             "Project monitor request successful",

+             "packages",

+         )

+     finally:

+         checkpoint("Streaming prepared")

@@ -21,6 +21,7 @@ 

  from coprs.logic.complex_logic import ComplexLogic

  from coprs.logic.users_logic import UsersLogic

  from coprs.exceptions import ObjectNotFound

+ from coprs.measure import checkpoint_start

  

  

  def create_user_wrapper(username, email, timezone=None):
@@ -62,12 +63,17 @@ 

  

  

  @app.before_request

- def set_empty_user():

-     flask.g.user = None

+ def before_request():

+     """

+     Configure some useful defaults for (before) each request.

+     """

  

+     # Checkpoints initialization

+     checkpoint_start()

  

- @app.before_request

- def lookup_current_user():

+     # Load the logged-in user, if any.

+     # https://github.com/PyCQA/pylint/issues/3793

+     # pylint: disable=assigning-non-slot

      flask.g.user = username = None

      if "openid" in flask.session:

          username = fed_raw_name(flask.session["openid"])

@@ -323,6 +323,19 @@ 

          assert resp.status_code == 200

          return json.loads(resp.data)

  

+     def fail_source_build(self, build_id):

+         """ Mimic backend marking source build as failed """

+         form_data = {

+             "builds": [{

+                 "id": int(build_id),

+                 "task_id": str(build_id),

+                 "srpm_url": "http://foo",

+                 "status": 0,

+             }],

+         }

+         assert self.update(form_data).status_code == 200

+ 

+ 

      def finish_build(self, build_id, package_name=None):

          """

          Given the build_id, finish the build with succeeded state

@@ -0,0 +1,192 @@ 

+ """

+ Test /api_3/monitor

+ """

+ 

+ import pytest

+ 

+ from tests.coprs_test_case import CoprsTestCase, TransactionDecorator

+ 

+ 

+ class TestAPIv3Monitor(CoprsTestCase):

+     @TransactionDecorator("u1")

+     @pytest.mark.usefixtures("f_users", "f_users_api", "f_coprs",

+                              "f_mock_chroots", "f_builds", "f_db")

+     @pytest.mark.parametrize("case", [

+         {"project_dirname": "nonexistent"},  # non-existing dir

+         {"project_dirname": "foocopr"},      # main dir

+         {"additional_fields[]": ["url_build_log", "url_backend_log"]},

+         {"additional_fields[]": ["wrongarg1", "wrongarg2"]},

+     ])

+     def test_v3_monitor(self, case):

+         params = {

+             "ownername": "user1",

+             "projectname": "foocopr",

+         }

+         params.update(case)

+         result = self.tc.get("/api_3/monitor", query_string=params)

+         if case.get("project_dirname") == "nonexistent":

+             assert result.status_code == 404

+             assert "'nonexistent' doesn't exist in 'user1/" in result.json["error"]

+             return

+ 

+         if case.get("additional_fields[]") and \

+                     "wrongarg1" in case["additional_fields[]"]:

+             assert result.status_code == 400

+             assert result.json["error"] == \

+                    "Wrong additional_fields argument(s): wrongarg1, wrongarg2"

+             return

+ 

+         self.api3.create_distgit_package("foocopr", "cpio")

+         self.api3.rebuild_package("foocopr", "cpio")

+         self.backend.finish_build(5)

+ 

+         assert self.tc.get("/api_3/monitor", query_string=params).json \

+                == {

+             "message": "Project monitor request successful",

+             "output": "ok",

+             "packages": [{

+                 "name": "cpio",

+                 "chroots": {

+                     "fedora-18-x86_64": {

+                             "build_id": 5,

+                             "state": "succeeded",

+                             "status": 1,

+                     } | ({

+                             "url_backend_log": (

+                                 "http://copr-be-dev.cloud.fedoraproject.org/"

+                                 "results/user1/foocopr/fedora-18-x86_64/xyz/"

+                                 "backend.log.gz"),

+                             "url_build_log": (

+                                 "http://copr-be-dev.cloud.fedoraproject.org/"

+                                 "results/user1/foocopr/fedora-18-x86_64/xyz/"

+                                 "builder-live.log.gz"),

+                     } if "additional_fields[]" in case else {})

+                 },

+             }, {

+                 "name": "hello-world",

+                 "chroots": {

+                     "fedora-18-x86_64": {

+                         "build_id": 2,

+                         "state": "waiting",

+                         "status": 9,

+                     } | ({

+                         "url_backend_log": None,

+                         "url_build_log": None,

+                     } if "additional_fields[]" in case else {})

+                 },

+             }]

+         }

+ 

+     @TransactionDecorator("u1")

+     @pytest.mark.usefixtures("f_users", "f_users_api", "f_mock_chroots", "f_db")

+     def test_v3_monitor_empty_project(self):

+         self.web_ui.new_project(

+             "test",

+             ["fedora-rawhide-i386", "fedora-18-x86_64"])

+         self.web_ui.create_distgit_package("test", "tar")

+         result = self.tc.get("/api_3/monitor", query_string={

+             "ownername": "user1",

+             "projectname": "test",

+         })

+         assert result.json == {

+             "message": "Project monitor request successful",

+             "output": "ok",

+             "packages": [],

+         }

+ 

+ 

+     @TransactionDecorator("u1")

+     @pytest.mark.usefixtures("f_users", "f_users_api", "f_mock_chroots", "f_db")

+     def test_v3_monitor_multi_chroot(self):

+         self.web_ui.new_project(

+             "test",

+             ["fedora-rawhide-i386", "fedora-18-x86_64"])

+         self.web_ui.create_distgit_package("test", "tar")

+         self.api3.rebuild_package("test", "tar")

+         self.backend.finish_build(1, package_name="tar")

+ 

+         assert self.tc.get("/api_3/monitor", query_string={

+             "ownername": "user1",

+             "projectname": "test",

+         }).json == {

+             "message": "Project monitor request successful",

+             "output": "ok",

+             "packages": [{

+                 "name": "tar",

+                 "chroots": {

+                     "fedora-18-x86_64": {

+                         "build_id": 1,

+                         "state": "succeeded",

+                         "status": 1,

+                     },

+                     "fedora-rawhide-i386": {

+                         "build_id": 1,

+                         "state": "succeeded",

+                         "status": 1,

+                     },

+                 },

+             }]

+         }

+ 

+     @TransactionDecorator("u1")

+     @pytest.mark.usefixtures("f_users", "f_users_api", "f_mock_chroots", "f_db")

+     def test_v3_monitor_source_build(self):

+         self.web_ui.new_project(

+             "test",

+             ["fedora-rawhide-i386", "fedora-18-x86_64"])

+         self.web_ui.create_distgit_package("test", "tar")

+         self.api3.rebuild_package("test", "tar")

+ 

+         def _fixup_result(result_dict, update=None):

+             for package in result_dict["packages"]:

+                 for _, chroot in package["chroots"].items():

+                     if update:

+                         chroot.update(update)

+ 

+         result = self.tc.get("/api_3/monitor", query_string={

+             "ownername": "user1",

+             "projectname": "test",

+             "additional_fields[]": ["url_build_log"],

+         }).json

+         _fixup_result(result)

+ 

+         expected_result = {

+             'message': 'Project monitor request successful',

+             'output': 'ok',

+             'packages': [{

+                 "name": "tar",

+                 'chroots': {

+                     'fedora-rawhide-i386': {

+                         'build_id': 1,

+                         'state': 'waiting',

+                         'status': 9,

+                         # we don't have build log here

+                         'url_build_log': None,

+                     },

+                     'fedora-18-x86_64': {

+                         'build_id': 1,

+                         'state': 'waiting',

+                         'status': 9,

+                         # we don't have build log here

+                         'url_build_log': None,

+                     },

+                 },

+             }]

+         }

+         assert result == expected_result

+ 

+         # finish with source failure

+         self.backend.fail_source_build(1)

+         result = self.tc.get("/api_3/monitor", query_string={

+             "ownername": "user1",

+             "projectname": "test",

+             "additional_fields[]": ["url_build_log"],

+         }).json

+         _fixup_result(result)

+ 

+         # We expect that the result is failed!

+         _fixup_result(expected_result, update={

+             "status": 0,

+             "state": "failed",

+         })

+         assert result == expected_result

@@ -63,13 +63,21 @@ 

          build = self.models.Build.query.get(1)

          assert json.loads(build.source_json) == expected_source_dict

          assert build.source_type == BuildSourceEnum(source_type_text)

-         assert build.chroots == []

  

+         def _assert_default_chroots(test_build):

+             # We assign Package to Build as soon as possible, and at the same

+             # time we allocate BuildChroots.

+             assert {bch.name for bch in test_build.chroots} == {

+                 "fedora-17-x86_64",

+                 "fedora-17-i386",

+             }

+ 

+         _assert_default_chroots(build)

          rebuild_data["chroots"] = chroots

          self.post_api3_with_auth(endpoint, rebuild_data, user)

          build = self.models.Build.query.get(2)

          assert json.loads(build.source_json) == expected_source_dict

          if "fedora-18-x86_64" in chroots or chroots == []:

-             assert build.chroots == []

+             _assert_default_chroots(build)

          else:

              assert [mch.name for mch in build.chroots] == ["fedora-17-i386"]

@@ -27,32 +27,33 @@ 

  class TestMonitor(CoprsTestCase):

  

      @new_app_context

-     def test_regression_monitor_no_copr_returned(self, f_db, f_users, f_mock_chroots):

+     @pytest.mark.usefixtures("f_db", "f_users", "f_mock_chroots", "f_db")

+     def test_regression_monitor_no_copr_returned(self):

          # https://bugzilla.redhat.com/show_bug.cgi?id=1165284

- 

-         # commit users to the database

-         self.db.session.commit()

          copr_name = u"temp"

  

          # trying to get monitor page for non-existing project

-         res = self.tc.get("/coprs/{}/{}/monitor/".format(self.u1.name, copr_name))

-         assert res.status_code == 404

<