From 2cc4f94c59d23bdf7a65336084c6c38b46c9f15d Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Apr 06 2021 10:43:25 +0000 Subject: Setup a periodic job to clean too old koji tasks --- diff --git a/files/clean-koji-stalled-tasks.py b/files/clean-koji-stalled-tasks.py new file mode 100755 index 0000000..569c33a --- /dev/null +++ b/files/clean-koji-stalled-tasks.py @@ -0,0 +1,125 @@ +#!/bin/env python3 + +from typing import List, Optional +from subprocess import run, CalledProcessError +from datetime import datetime, timedelta +import unittest +import argparse +import sys + +KOJI_CMD = ["koji"] +MAX_HR = 12 + +mocked_list_tasks_ids = """ +64642854 20 zuul OPEN noarch build (rawhide, libkcapi-1.2.1-1.fc35.src.rpm) +64642908 19 zuul OPEN ppc64le +buildArch (libkcapi-1.2.1-1.fc35.src.rpm, ppc64le) +64642862 20 zuul OPEN noarch build (rawhide, libkcapi-1.2.1-1.fc35.src.rpm) +""" + +mocked_task_info_1 = """ +Task: 64642801 +Type: build +Owner: zuul +State: open +Created: Fri Mar 26 15:43:39 2021 +Started: Thu Apr 1 21:39:23 2021 +Host: buildvm-s390x-11.s390.fedoraproject.org +""" + +now = datetime.strftime(datetime.now(), "%c") +mocked_task_info_2 = """ +Task: 64642802 +Type: build +Owner: zuul +State: open +Created: %s +Started: %s +Host: buildvm-s390x-11.s390.fedoraproject.org +""" % ( + now, + now, +) + + +def call_koji_cmd(args: List[str], mock: bool = False) -> Optional[str]: + if mock and args == ["list-tasks", "--quiet", "--mine"]: + return mocked_list_tasks_ids + if mock and args == ["taskinfo", "64642801"]: + return mocked_task_info_1 + if mock and args == ["taskinfo", "64642802"]: + return mocked_task_info_2 + cmd = KOJI_CMD + args + compp = run(cmd, capture_output=True, text=True) + try: + compp.check_returncode() + except CalledProcessError as exc: + print("Command %s did not success: %s" % (" ".join(cmd), exc)) + return None + return compp.stdout + + +def get_tasks_ids_list(mocked: bool = False) -> Optional[List[str]]: + cmd_output = call_koji_cmd(["list-tasks", "--quiet", "--mine"], mock=mocked) + if not cmd_output: + return None + return list( + map(lambda l: l.split()[0], filter(lambda x: x, cmd_output.splitlines())) + ) + + +def check_stalled_task(task_id: str, mocked: bool = False) -> bool: + cmd_output = call_koji_cmd(["taskinfo", task_id], mock=mocked) + if not cmd_output: + return False + started_line = list( + filter( + lambda l: l.startswith("Started: "), + filter(lambda x: x, cmd_output.splitlines()), + ) + ) + if not started_line: + return False + started_date = datetime.strptime(started_line[0].replace("Started: ", ""), "%c") + stalled = started_date < datetime.now() - timedelta(hours=MAX_HR) + print("Tasks %s started at %s [stalled: %s]" % (task_id, started_date, stalled)) + return stalled + + +def run_check(noop: bool) -> None: + tasks_ids = get_tasks_ids_list() + if tasks_ids: + for task_id in tasks_ids: + if not check_stalled_task(task_id): + continue + if not noop: + print("Cancelling task %s ..." % task_id) + call_koji_cmd(["cancel", task_id]) + else: + print("Will cancel task %s ..." % task_id) + + +### Run tests with python -m unittest clean-koji-stalled-tasks.py ### +class ToolTests(unittest.TestCase): + def test_get_tasks_ids_list(self): + tasks_ids = get_tasks_ids_list(mocked=True) + self.assertListEqual(["64642854", "64642908", "64642862"], tasks_ids) + + def test_check_stalled_tasks(self): + self.assertTrue(check_stalled_task("64642801", mocked=True)) + self.assertFalse(check_stalled_task("64642802", mocked=True)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--noop", help="Do not cancel tasks (only report)", action="store_true" + ) + if "-m unittest" in sys.argv[0]: + # Skip main if running local tests + pass + else: + args = parser.parse_args() + run_check(args.noop) + + +main() diff --git a/playbooks/koji/clean-stalled-koji-tasks.yaml b/playbooks/koji/clean-stalled-koji-tasks.yaml new file mode 100644 index 0000000..5432c72 --- /dev/null +++ b/playbooks/koji/clean-stalled-koji-tasks.yaml @@ -0,0 +1,6 @@ +- hosts: all + tasks: + - include_role: + name: krb-secret + - name: Run cleaning script + script: files/clean-koji-stalled-tasks.py diff --git a/zuul.d/jobs.dhall b/zuul.d/jobs.dhall index 62a5512..101c64b 100644 --- a/zuul.d/jobs.dhall +++ b/zuul.d/jobs.dhall @@ -107,6 +107,18 @@ let check_for_arches = , nodeset = Some default_nodeset } +let clean_stalled_koji_tasks = + Zuul.Job::{ + , name = "${job-prefix}clean-stalled-koji-tasks" + , protected = Some True + , description = Some "Cancel Koji tasks running for more than 12 hours" + , nodeset = Some default_nodeset + , roles = Some [ { zuul = "zuul-distro-jobs" } ] + , run = Some "playbooks/koji/clean-stalled-koji-tasks.yaml" + , secrets = Some + [ Zuul.Job.Secret::{ name = "krb_keytab", secret = "krb_keytab" } ] + } + let common_koji_rpm_build = Zuul.Job::{ , name = "${job-prefix}common-koji-rpm-build" @@ -199,7 +211,11 @@ let generateRpmBuildJobs Branches.all let Jobs = - [ check_for_tests, check_for_arches, common_koji_rpm_build ] + [ clean_stalled_koji_tasks + , check_for_tests + , check_for_arches + , common_koji_rpm_build + ] # generateRpmBuildJobs KojiBuild.Type.Scratch # generateRpmBuildJobs KojiBuild.Type.Final diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 79996e0..406ac4c 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -1,6 +1,18 @@ # Code generated by dhall-to-yaml. DO NOT EDIT. - job: + name: clean-stalled-koji-tasks + description: Cancel Koji tasks running for more than 12 hours + run: playbooks/koji/clean-stalled-koji-tasks.yaml + nodeset: fedora-33-container + protected: true + roles: + - zuul: zuul-distro-jobs + secrets: + - name: krb_keytab + secret: krb_keytab + +- job: name: check-for-tests description: Check the project has a tests/tests.yml run: playbooks/rpm/check-for-tests.yaml diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index ab8f008..880181c 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -99,6 +99,9 @@ # Linter job for the current repository - project: + periodic: + jobs: + - clean-stalled-koji-tasks check: jobs: - linters: