#30 bring production up to date with master
Merged 3 years ago by scoady. Opened 3 years ago by scoady.
fedora-ci/ scoady/monitor-gating production  into  production

file modified
+7 -3
@@ -1,7 +1,7 @@ 

  # This Dockerfile is used to build and run monitor-gating on Openshift

  FROM fedora:31

  

- LABEL maintainer "Pierre-Yves Chibon <pingou@pingoured.fr>"

+ LABEL maintainer "Fedora Infrastructure Team <infrastructure@lists.fedoraproject.org>"

  

  RUN dnf -y install python3-requests bodhi-client fedpkg fedpkg-stage \

      python3-toml git python3-koji python3-fedora-messaging cracklib-dicts \
@@ -9,9 +9,12 @@ 

  

  COPY . /opt/code

  

- RUN echo "packagerbot" > /.fedora.upn \

+ RUN cd /opt/code && python3 setup.py install && mkdir /opt/config && cp runner.cfg /opt/config \

+     && echo "packagerbot" > /.fedora.upn \

      && chgrp -R 0 /opt/code \

      && chmod -R g=u /opt/code \

+     && chgrp -R 0 /opt/config \

+     && chmod -R g=u /opt/config \

      && chgrp -R 0 /.ssh \

      && chmod -R g=u /.ssh \

      && chgrp -R 0 /.fedora \
@@ -22,4 +25,5 @@ 

  

  USER 1000

  WORKDIR /

- ENTRYPOINT ["sh", "/opt/code/entrypoint.sh"]

+ ENTRYPOINT ["/opt/code/entrypoint.sh"]

+ CMD ["monitor-gating", "/opt/config/runner.cfg"]

file modified
+3
@@ -6,6 +6,7 @@ 

    fi

  fi

  

+ 

  ln -s /opt/ssh/id_rsa /.ssh/id_rsa || true

  if [ -z ${PRODUCTION+x} ]; then

  # Staging info
@@ -17,4 +18,6 @@ 

  ssh-keyscan pkgs.fedoraproject.org >> /.ssh/known_hosts

  fi

  

+ klist -t -k /etc/keytabs/monitor-gating-keytab

+ 

  python3 /opt/code/runner.py /opt/config/runner.cfg

empty or binary file added
monitor_gating/clean_up_side_tags.py clean_up_side_tags.py
file renamed
+16 -18
@@ -14,12 +14,12 @@ 

  def get_cli_args(args):

  

      parser = argparse.ArgumentParser(

-         prog="clean up side-tags", formatter_class=argparse.ArgumentDefaultsHelpFormatter

+         prog="clean up side-tags",

+         formatter_class=argparse.ArgumentDefaultsHelpFormatter,

      )

  

      parser.add_argument(

-         "conf",

-         help="Configuration file used by the runner",

+         "conf", help="Configuration file used by the runner",

      )

  

      return parser.parse_args(args)
@@ -33,7 +33,9 @@ 

      try:

          output = subprocess.check_output(command, stderr=subprocess.PIPE)

      except subprocess.CalledProcessError as e:

-         _log.error("Command `{}` return code: `{}`".format(" ".join(command), e.returncode))

+         _log.error(

+             "Command `{}` return code: `{}`".format(" ".join(command), e.returncode)

+         )

          _log.error("stdout:\n-------\n{}".format(e.stdout))

          _log.error("stderr:\n-------\n{}".format(e.stderr))

          pass
@@ -41,18 +43,20 @@ 

      return output

  

  

- def main(args):

+ def main():

+     """ Main method. """

+     args = get_cli_args(sys.argv[1:])

  

      conf = toml.load(args.conf)

      if conf.get("kb_principal") and conf.get("kb_keytab_file"):

-         print(f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}")

+         print(

+             f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}"

+         )

          cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]]

          run_command(cmd)

  

      # list side-tags:

-     cmd = [

-         conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"].rsplit('@', 1)[0]

-     ]

+     cmd = [conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"]]

      output = run_command(cmd)

  

      if not output:
@@ -63,17 +67,11 @@ 

      # Remove all the side-tags but the last one (which is the latest one), which

      # we keep in case there is a run ongoing

      for line in output[:-1]:

-         side_tag = line.split('\t')[0]

+         side_tag = line.split("\t")[0]

          print("Removing side-tag: %s" % side_tag)

-         cmd = [

-             conf["fedpkg"], "remove-side-tag", side_tag

-         ]

+         cmd = [conf["fedpkg"], "remove-side-tag", side_tag]

          output = run_command(cmd)

  

  

- 

  if __name__ == "__main__":

-     """ Main method. """

- 

-     args = get_cli_args(sys.argv[1:])

-     main(args)

+     main()

monitor_gating/multi_builds.py monitor_gating_multi_builds.py
file renamed
+21 -6
@@ -30,7 +30,7 @@ 

  

  import toml

  

- from utils import MonitoringUtils

+ from .utils import MonitoringUtils

  

  _log = logging.getLogger(__name__)

  
@@ -66,14 +66,15 @@ 

      return parser.parse_args(args)

  

  

- def main(args):

+ def main(args, utils=None):

      """ Main method used by this script. """

      start = datetime.datetime.utcnow()

  

      args = get_arguments(args)

  

      conf = toml.load(args.conf)

-     utils = MonitoringUtils()

+     if utils is None:

+         utils = MonitoringUtils()

  

      with tempfile.TemporaryDirectory(prefix="ci-test-") as folder:

          print(f"Working in {folder}\n")
@@ -81,12 +82,26 @@ 

  

          # Bump the release on both packages:

          nevrs, side_tag_name = utils.clone_and_bump(

-             folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True

+             folder, nevrs, conf, conf["name_multi_1"], version=0, new_side_tag=True

          )

+         # Store the side_tag_name so we can use it in the runner

+         utils.side_tag_name = side_tag_name

+ 

          nevrs, _ = utils.clone_and_bump(

-             folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name

+             folder, nevrs, conf, conf["name_multi_2"], version=0, target=side_tag_name

          )

  

+         version, synced = utils.nevrs_synced(nevrs, conf)

+ 

+         if not synced:

+             nevrs, side_tag_name = utils.clone_and_bump(

+                 folder, nevrs, conf, conf["name_multi_1"], version=version, new_side_tag=True

+             )

+ 

+             nevrs, _ = utils.clone_and_bump(

+                 folder, nevrs, conf, conf["name_multi_2"], version=version, target=side_tag_name

+             )

+ 

          # Chain-build the packages

          utils.chain_build_packages(

              conf["fedpkg"],
@@ -219,7 +234,7 @@ 

              )

  

      utils.finalize(start)

-     return utils.logs

+     return utils

  

  

  if __name__ == "__main__":

file renamed
+82 -27
@@ -9,16 +9,17 @@ 

  import sched

  import sys

  import time

- import utils

  import uuid

  

  import fedora_messaging.api

  import fedora_messaging.exceptions

+ 

  import toml

  

- import monitor_gating_single_build

- import monitor_gating_multi_builds

- from utils import run_command

+ from . import multi_builds

+ from . import single_build

+ from .utils import MonitoringUtils, blocking_issues, run_command, report_failure

+ 

  

  s = sched.scheduler(time.time, time.sleep)

  conf = toml.load
@@ -44,8 +45,21 @@ 

          print(f"Fedora Messaging broker rejected message {msg.id}: {err}")

      except fedora_messaging.exceptions.ConnectionException as err:

          print(f"Error sending message {msg.id}: {err}")

+         print(f"topic: {topic}")

+         print(f"message: {message}")

      except Exception as err:

          print(f"Error sending fedora-messaging message: {err}")

+         print(f"topic: {topic}")

+         print(f"message: {message}")

+ 

+ 

+ def _clean_up_side_tags(utils):

+     try:

+         print("  Removing side-tag: %s" % utils.side_tag_name)

+         cmd = [conf["fedpkg"], "remove-side-tag", utils.side_tag_name]

+         run_command(cmd)

+     except Exception:

+         pass

  

  

  def schedule(conf):
@@ -57,8 +71,10 @@ 

          run_command(cmd)

  

      delay = conf["delay"]

-     delay_when_failing = conf["delay_when_failing"]

-     blocker_tags = conf["blocker_tags"]

+     report_project = conf["pagure_report_project"]

+     report_api_token = conf["pagure_api_token"]

+     report_env = conf["env"]

+ 

      print("Tests started:", datetime.datetime.utcnow(), flush=True)

      runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}"

      try:
@@ -66,43 +82,66 @@ 

          single_args = conf["workflow_single_gating_args"].split()

          notify(

              topic=f"single-build.start",

-             message={"arguments": single_args, "runid": runid,},

+             message={"arguments": single_args, "runid": runid},

          )

-         output = monitor_gating_single_build.main(single_args)

-         output_text = "\n".join(output)

+         monit_utils = single_build.main(single_args)

+         output_text = "\n".join(monit_utils.logs)

          if "[FAILED]" not in output_text:

              result = "succeeded"

          else:

              result = "failed"

+             report_failure(

+                 report_project,

+                 report_api_token,

+                 report_env,

+                 "single-package",

+                 monit_utils,

+             )

+ 

          notify(

              topic=f"single-build.end.{result}",

              message={

-                 "output": output,

+                 "output": monit_utils.logs,

                  "output_text": output_text,

                  "result": result,

                  "runid": runid,

+                 "failed": monit_utils.failed,

              },

          )

  

          # Multi Build Gating

          multi_args = conf["workflow_multi_gating_args"].split()

+         monit_utils = MonitoringUtils()

          notify(

              topic=f"multi-build.start",

-             message={"arguments": multi_args, "runid": runid,},

+             message={"arguments": multi_args, "runid": runid},

          )

-         output = monitor_gating_multi_builds.main(multi_args)

-         output_text = "\n".join(output)

+         try:

+             monit_utils = multi_builds.main(multi_args, utils=monit_utils)

+         finally:

+             _clean_up_side_tags(monit_utils)

+ 

+         output_text = "\n".join(monit_utils.logs)

          if "[FAILED]" not in output_text:

              result = "succeeded"

          else:

              result = "failed"

+             report_failure(

+                 report_project,

+                 report_api_token,

+                 report_env,

+                 "multi-package",

+                 monit_utils,

+             )

+ 

          notify(

              topic=f"multi-build.end.{result}",

              message={

-                 "output": output,

+                 "output": monit_utils.logs,

                  "output_text": output_text,

                  "result": result,

                  "runid": runid,

+                 "failed": monit_utils.failed,

              },

          )

  
@@ -110,13 +149,29 @@ 

      except Exception as err:

          print(f"Tests failed with: {err}", flush=True)

          print(sys.exc_info()[0])

+         print("-"*60)

+         print(sys.exc_info())

+         print("="*60)

+         import traceback

+         traceback.print_exc(file=sys.stdout)

+         print("-"*60)

+         traceback.print_stack()

+ 

  

-     blocking_issues = utils.blocking_issues(blocker_tags)

+         notify(

+             topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)},

+         )

+ 

+     delay_when_failing = conf["delay_when_failing"]

+     blocker_tags = conf["blocker_tags"]

+     blocking_project = conf["pagure_blocking_project"]

+ 

+     blocking_issues_list = blocking_issues(blocking_project, blocker_tags)

      now = datetime.datetime.utcnow().strftime("%H:%M:%S")

-     if blocking_issues:

+     if blocking_issues_list:

          print(

              f"{now} Next run in: {delay_when_failing} seconds because of "

-             f"{len(blocking_issues)} open issues",

+             f"{len(blocking_issues_list)} open issues",

              flush=True,

          )

          s.enter(delay_when_failing, 1, schedule, argument=(conf,))
@@ -125,17 +180,13 @@ 

          s.enter(delay, 1, schedule, argument=(conf,))

  

  

- def main(args):

-     """ Schedule the first test and run the scheduler. """

-     args = get_arguments(args)

-     conf = toml.load(args.conf)

-     s.enter(0, 1, schedule, argument=(conf,))

-     s.run()

- 

- 

- if __name__ == "__main__":

+ def main():

+     """ Main method. """

      try:

-         main(sys.argv[1:])

+         args = get_arguments(sys.argv[1:])

+         conf = toml.load(args.conf)

+         s.enter(0, 1, schedule, argument=(conf,))

+         s.run()

      except KeyboardInterrupt:

          from code import InteractiveConsole

  
@@ -143,3 +194,7 @@ 

              "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit",

              "LEAVING THE DEBUG CONSOLE",

          )

+ 

+ 

+ if __name__ == "__main__":

+     main()

monitor_gating/single_build.py monitor_gating_single_build.py
file renamed
+11 -3
@@ -30,7 +30,7 @@ 

  

  import toml

  

- from utils import MonitoringUtils

+ from .utils import MonitoringUtils

  

  _log = logging.getLogger(__name__)

  
@@ -102,7 +102,7 @@ 

              )

              gitfolder = os.path.join(folder, name)

              utils.switch_branch(conf["fedpkg"], branch, folder=gitfolder)

-             utils.bump_release(name, folder=gitfolder)

+             utils.bump_release(name, version=0, folder=gitfolder)

              utils.commit_changes("Bump release", folder=gitfolder)

              nevr = utils.get_nevr(conf["fedpkg"], folder=gitfolder)

              print(f"   Upcoming build : {nevr}")
@@ -232,6 +232,9 @@ 

              topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running",

              nevr=nevr,

          )

+ 

+         start_dg = datetime.datetime.utcnow()

+ 

          # Check at the CI pipeline has completed

          utils.lookup_results_datagrepper(

              base_url=conf["datagrepper"],
@@ -239,6 +242,7 @@ 

              topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error",

              nevr=nevr,

              duration=30,

+             start=start_dg,

          )

  

          # Check the tag of the build
@@ -257,6 +261,8 @@ 

              name="resultsdb",

              topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new",

              nevr=nevr,

+             duration=45,

+             start=start_dg,

          )

  

          # Check that greenwave reacted to resultsdb's new results
@@ -265,6 +271,8 @@ 

              name="greenwave",

              topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update",

              nevr=nevr,

+             duration=60,

+             start=start_dg,

          )

  

          # Check the tag of the build -- build is blocked but should be signed
@@ -303,7 +311,7 @@ 

              )

  

      utils.finalize(start)

-     return utils.logs

+     return utils

  

  

  if __name__ == "__main__":

file renamed
+122 -31
@@ -11,33 +11,67 @@ 

  import os

  import subprocess

  import time

+ import re

  

  import requests

  

  _log = logging.getLogger(__name__)

  

  

- def blocking_issues(tags):

+ def report_failure(project, token, env, workflow, monit_utils):

+     """ Open a pagure ticket against the instance specified in the

+     configuration file when something does not work.

+     """

+     url = f"https://pagure.io/api/0/{project}/new_issue"

+     title = f"Failure in {env} of the {workflow} packager workflow"

+     logs = "\n".join(monit_utils.logs)

+     content = f"""A run of monitor-gating has just failed in {env} for the {workflow} workflow.

+ 

+ The suspects are '{", ".join(monit_utils.failed)}'.

+ 

+ Full log:

+ ````

+ {logs}

+ ````

+ """

+     tag = env

+ 

+     data = {

+         "title": title,

+         "content": content,

+         "tag": tag,

+     }

+     headers = {

+         "Authorization": f"token {token}",

+     }

+ 

+     req = requests.post(url, data=data, headers=headers)

+     if not req.ok:

+         print(f"Error when trying to open a ticket at: {url} to report the failure", flush=True)

+ 

+ 

+ def blocking_issues(project, tags):

      """Lists blocking issues we track in the fedora-infrastructure project.

      """

      if not tags:

          print(f"No tags to filter blocking issues by, returning empty.")

          return []

-     api = f"https://pagure.io/api/0/fedora-infrastructure/issues"

-     q = f"?status=Open&tags={tags[0]}"

+ 

+     api = f"https://pagure.io/api/0/{project}/issues?status=Open&tags={tags[0]}"

      issues = []

      try:

-         r = requests.get(api + q)

-         issues = r.json()["issues"]

-         if tags:

-             t = set(tags[1:])

-             issues = [i for i in issues if t & set(i["tags"])]

-         for i in issues:

-             print(

-                 f"Found blocking issue https://pagure.io/fedora-infrastructure/issue/{i['id']}"

-             )

+         r = requests.get(api)

+         if not r.ok:

+             print(f"Failed to query: {api} -- returned : {r.status_code}", flush=True)

+         else:

+             issues = r.json()["issues"]

+             if tags:

+                 t = set(tags[1:])

+                 issues = [i for i in issues if t & set(i["tags"])]

+             for i in issues:

+                 print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}", flush=True)

      except Exception as e:

-         print(f"Error when querying pagure for blocking issues: {e}")

+         print(f"Error when querying pagure for blocking issues: {e}", flush=True)

      return issues

  

  
@@ -49,6 +83,7 @@ 

      def __init__(self):

          """ Instanciate the object. """

          self.logs = []

+         self.failed = []

  

      def print_user(self, content, success=None):

          """ Prints the specified content to the user.
@@ -81,17 +116,24 @@ 

                  [command, "--user", username, "clone", f"{namespace}/{name}"],

                  cwd=folder,

              )

-             clone_folder = os.path.join(folder, name)

-             run_command(

-                 ["git", "config", "user.name", "packagerbot"], cwd=clone_folder,

-             )

-             run_command(

-                 ["git", "config", "user.email", "admin@fedoraproject.org"],

-                 cwd=clone_folder,

-             )

-             self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("git/dist-git")

              self.print_user(info_log, success=False)

+         else:

+             try:

+                 # Assume these commands can't fail

+                 clone_folder = os.path.join(folder, name)

+                 run_command(

+                     ["git", "config", "user.name", "packagerbot"], cwd=clone_folder,

+                 )

+                 run_command(

+                     ["git", "config", "user.email", "admin@fedoraproject.org"],

+                     cwd=clone_folder,

+                 )

+                 self.print_user(info_log, success=True)

+             except MonitoringException:

+                 self.failed.append("git")

+                 self.print_user(info_log, success=False)

  

      def add_remote(self, name, url, folder):

          """ Add the specified remote to the git repo in the folder with the
@@ -103,6 +145,7 @@ 

              run_command(["git", "remote", "add", name, url], cwd=folder)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("git")

              self.print_user(info_log, success=False)

  

      def switch_branch(self, command, name, folder):
@@ -114,17 +157,23 @@ 

              run_command([command, "switch-branch", f"{name}"], cwd=folder)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("fedpkg")

              self.print_user(info_log, success=False)

  

-     def bump_release(self, name, folder):

+     def bump_release(self, name, version, folder):

          """ Bump the release of the spec file the specified git repo.

          """

          info_log = f"Bumping release of: {name}.spec"

          self.print_user(info_log)

          try:

-             run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder)

-             self.print_user(info_log, success=True)

+             if version:

+                 run_command(["rpmdev-bumpspec", f"{name}.spec", "-n", f"{version}"], cwd=folder)

+                 self.print_user(info_log, success=True)

+             else:

+                 run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder)

+                 self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("rpmdev-bumspec")

              self.print_user(info_log, success=False)

  

      def commit_changes(self, commit_log, folder):
@@ -137,6 +186,7 @@ 

              run_command(["git", "commit", "-asm", commit_log], cwd=folder)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("git")

              self.print_user(info_log, success=False)

  

      def push_changes(self, folder, target, branch, force=False):
@@ -151,6 +201,7 @@ 

              run_command(cmd, cwd=folder)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("git/dist-git")

              self.print_user(info_log, success=False)

  

      def pull_changes(self, folder, target, branch):
@@ -163,6 +214,7 @@ 

              run_command(cmd, cwd=folder)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("git/dist-git")

              self.print_user(info_log, success=False)

  

      def open_pullrequest(self, base_url, username, namespace, name, branch, token):
@@ -172,7 +224,7 @@ 

          info_log = f"Creating PR from forks/{username}/{namespace}/{name}"

          self.print_user(info_log)

          url = "/".join(

-             [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new",]

+             [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new"]

          )

          data = {

              "branch_to": branch,
@@ -190,6 +242,7 @@ 

              success = False

              pr_id = None

              pr_uid = None

+             self.failed.append("dist-git")

          else:

              output = req.json()

              pr_id = str(output["id"])
@@ -212,6 +265,7 @@ 

              self.print_user(info_log, success=True)

              return nevr.strip().decode("utf-8")

          except MonitoringException:

+             self.failed.append("fedpkg")

              self.print_user(info_log, success=False)

  

      def build_package(self, command, folder, target=None):
@@ -226,6 +280,7 @@ 

              run_command(command, cwd=folder)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("koji")

              self.print_user(info_log, success=False)

  

      def chain_build_packages(self, command, packages, folder, target=None):
@@ -245,6 +300,7 @@ 

              run_command(command, cwd=folder)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("koji")

              self.print_user(info_log, success=False)

  

      def get_build_tags(self, koji_url, nevr, expected_ends):
@@ -258,9 +314,7 @@ 

              "koji",

          ]

          if koji_url:

-             command.extend(

-                 ["-s", koji_url,]

-             )

+             command.extend(["-s", koji_url])

          command.extend(["call", "listTags", nevr])

  

          success = False
@@ -303,6 +357,8 @@ 

                  break

  

          info_log = f"Retrieving koji tags: {tags}"

+         if not success:

+             self.failed.append("koji")

          self.print_user(info_log, success=success)

  

      def create_update(
@@ -336,6 +392,7 @@ 

              run_command(command)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("bodhi")

              self.print_user(info_log, success=False)

  

      def get_update_id(self, nevr, url):
@@ -356,6 +413,7 @@ 

  

              if (datetime.datetime.utcnow() - start).seconds > (15 * 60):

                  success = False

+                 self.failed.append("bodhi")

                  info_log = f"Update for {nevr} not created within 15 minutes"

                  break

  
@@ -514,6 +572,8 @@ 

          end = datetime.datetime.utcnow()

          info_log += f" - ran for: {(end - start_lookup).seconds}s"

          self.print_user(info_log, success=success)

+         if not success:

+             self.failed.append("datagrepper")

  

      def lookup_ci_resultsdb(self, nevr, name, url):

          """ Check the CI results in the specified resultsdb for results about
@@ -563,6 +623,8 @@ 

          end = datetime.datetime.utcnow()

          info_log += f" - ran for: {(end - start).seconds}s"

          self.print_user(info_log, success=success)

+         if not success:

+             self.failed.append("resultsdb")

  

      def waive_update(self, command, updateid, prod=True, username=None, password=None):

          """ Waive all the tests results for the specified update using bodhi's
@@ -588,6 +650,7 @@ 

              run_command(command)

              self.print_user(info_log, success=True)

          except MonitoringException:

+             self.failed.append("waiverdb")

              self.print_user(info_log, success=False)

  

      def get_pr_flag(
@@ -642,6 +705,8 @@ 

                      break

  

          self.print_user(info_log, success=success)

+         if not success:

+             self.failed.append("dist-git")

  

      def merge_pr(self, base_url, username, namespace, name, pr_id, token):

          """ Merge the specified PR
@@ -661,6 +726,8 @@ 

              success = True

  

          self.print_user(info_log, success=success)

+         if not success:

+             self.failed.append("dist-git")

  

      def finalize(self, start):

          """ End data returned. """
@@ -683,11 +750,12 @@ 

              self.print_user(info_log, success=True)

          except (MonitoringException, Exception) as err:

              print(err)

+             self.failed.append("dist-git")

              self.print_user(info_log, success=False)

          return side_tag_name

  

      def clone_and_bump(

-         self, folder, nevrs, conf, name, target=None, new_side_tag=False

+         self, folder, nevrs, conf, name, version, target=None, new_side_tag=False

      ):

          """Clone the repo, bump the release, commit and push."""

          namespace = conf["namespace"]
@@ -703,7 +771,11 @@ 

          if new_side_tag:

              side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder)

              target = side_tag_name

-         self.bump_release(name, folder=gitfolder)

+         if version:

+             version += version

+             self.bump_release(name, version, folder=gitfolder)

+         else:

+             self.bump_release(name, version=0, folder=gitfolder)

          self.commit_changes("Bump release", folder=gitfolder)

          nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder)

          nevrs[name] = nevr
@@ -712,6 +784,25 @@ 

          print(f"   Upcoming build : {nevr}")

          return (nevrs, target)

  

+     def nevrs_synced(self, nevrs, conf):

+         package_one = conf["name_multi_1"]

+         package_two = conf["name_multi_2"]

+ 

+         nevr_one = nevrs[package_one]

+         nevr_two = nevrs[package_two]

+ 

+         if nevr_one.startswith(package_one):

+             release_one = nevr_one[len(package_one):]

+             version = int(re.search(r'\d+', nevr_one).group())

+ 

+         if nevr_two.startswith(package_two):

+             release_two = nevr_one[len(package_two):]

+ 

+         if release_one != release_two:

+             return version, False

+ 

+         return version, True

+ 

  

  def run_command(command, cwd=None):

      """ Run the specified command in a specific working directory if one

file added
+3
@@ -0,0 +1,3 @@ 

+ fedora_messaging

+ requests

+ toml

file modified
+21 -2
@@ -1,11 +1,11 @@ 

  # Time between two runs in second

+ # 3600 = 1h

  delay = 3600

  

  # Time between two blocked runs in second

+ # 43200 = 12h

  delay_when_failing = 43200

  

- # blocker issue tags, issue has to have all of them

- blocker_tags = ['packager_workflow_blocker', 'staging']

  

  # CLI arguments to give to the script testing the single build gating workflow

  workflow_single_gating_args = "--conf monitor_gating_stg.cfg --auto-update --no-pr"
@@ -20,3 +20,22 @@ 

  # kb_keytab_file = "/etc/keytabs/monitor-gating-keytab"

  

  fedpkg = "fedpkg"

+ 

+ # The configuration key below are used when interacting with pagure projects

+ # There are two ways monitor-gating interacts with them.

+ # a) it monitors a specific project to slows down its run in case a known issue

+ #    prevents the workflow from working (so as to now increase the load on a

+ #    known broken system).

+ # b) it reports to a specific project (but not necessarily the same) when a

+ #    run failed to run properly end to end.

+ 

+ # Project whose issue will slow down the subsequent runs (delay defined

+ # above).

+ pagure_blocking_project = "fedora-infrastructure"

+ # blocker issue tags, issue has to have all of them.

+ blocker_tags = ['packager_workflow_blocker', 'staging']

+ 

+ # Project against which failed runs report their failure.

+ pagure_report_project = "fedora-infra/packaging_workflow_health"

+ pagure_api_token = "<to edit>"

+ env = "prod"

file added
+66
@@ -0,0 +1,66 @@ 

+ import os

+ 

+ from setuptools import setup

+ 

+ 

+ def get_requirements():

+     requirements = []

+     with open("requirements.txt", "r") as f:

+         for line in f:

+             line = line.strip()

+             before, _, after = line.partition("#")

+             # Allow source control references for development.

+             if before.startswith("git+"):

+                 if not after:

+                     # The name is in the fragment

+                     continue

+                 _, _, requirement = after.rpartition("=")

+                 requirement, _, _ = requirement.partition("#")

+             else:

+                 requirement = before

+ 

+             requirement = requirement.strip()

+ 

+             if requirement:

+                 requirements.append(requirement)

+     return requirements

+ 

+ 

+ here = os.path.abspath(os.path.dirname(__file__))

+ with open(os.path.join(here, "README.rst"), "r") as f:

+     README = f.read()

+ 

+ 

+ setup(

+     name="monitor-gating",

+     version="0.0.1",

+     # Possible options are at https://pypi.python.org/pypi?%3Aaction=list_classifiers

+     classifiers=[

+         "Development Status :: 5 - Alpha",

+         "Intended Audience :: Developers",

+         "Intended Audience :: System Administrators",

+         # 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)',

+         "Operating System :: POSIX :: Linux",

+         "Programming Language :: Python :: 3",

+         "Programming Language :: Python :: 3.6",

+         "Programming Language :: Python :: 3.7",

+         "Topic :: System :: Software Distribution",

+     ],

+     # license=LICENSE,

+     maintainer="Fedora Infrastructure Team",

+     maintainer_email="infrastructure@lists.fedoraproject.org",

+     platforms=["Fedora", "GNU/Linux"],

+     url="https://pagure.io/fedora-ci/monitor-gating",

+     description="Monitor the health of gating in Fedora",

+     long_description=README,

+     keywords="fedora",

+     packages=["monitor_gating"],

+     include_package_data=True,

+     zip_safe=False,

+     install_requires=get_requirements(),

+     entry_points="""

+         [console_scripts]

+         monitor-gating = monitor_gating.runner:main

+         monitor-gating-clean-up-side-tags = monitor_gating.clean_up_side_tags:main

+     """,

+ )