From 667eb72684f1881b5b37dd0288fda5ba75dbcaf5 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jun 20 2019 16:48:31 +0000 Subject: moved to https://github.com/coreos/fedora-coreos-koji-tagger --- diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 9bce9f5..0000000 --- a/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -FROM registry.fedoraproject.org/fedora:30 - -# set PYTHONUNBUFFERED env var to non-empty string so that our -# periods with no newline get printed immediately to the screen -ENV PYTHONUNBUFFERED=true - -# Install pagure/fedmsg libraries -RUN dnf -y install python3-libpagure fedora-messaging koji krb5-workstation && dnf clean all - -# Grab the kerberos configuration. Pulling directly from upstream -# here rather than installing the fedora-packager rpm because it's -# a bunch of deps and we only need one file. -RUN curl -L https://pagure.io/fedora-packager/raw/master/f/krb-configs/fedoraproject_org > /etc/krb5.conf.d/fedoraproject_org - -RUN mkdir /work -WORKDIR /work - -# Copy the fedora config for fedora-messaging and also generate a random UUID -# https://fedora-messaging.readthedocs.io/en/latest/fedora-broker.html#getting-connected -# Note this will mean that if there is more than one container running -# using this image they will be reading from the same queue. Generally -# I expect this to only be running in one place. -RUN sed -e "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" /etc/fedora-messaging/fedora.toml > /work/my_config.toml - -# Set the Application Name -RUN sed -i 's|Example Application|CoreOS Koji Tagger: https://pagure.io/dusty/coreos-koji-tagger|' /work/my_config.toml - -# Lower log levels to WARNING level -RUN sed -i 's/INFO/WARNING/' /work/my_config.toml - -# Set the format for the log messages -RUN sed -i 's/format =.*$/format = "%(asctime)s %(levelname)s %(name)s - %(message)s"/' /work/my_config.toml - -# We only care about pungi.compose.status.change messages -RUN sed -i 's/^routing_keys.*$/routing_keys = ["io.pagure.prod.pagure.git.receive"]/' /work/my_config.toml - -# Put compose-tracker into a location that can be imported -ADD coreos_koji_tagger.py /usr/lib/python3.7/site-packages/ - -# Environment variable to be defined by the user that defines the -# filesystem path to the keytab file. If blank it will be ignored -# and privileged (write) operations won't be attempted -ENV COREOS_KOJI_TAGGER_KEYTAB_FILE '' - -# Error when trying to store the kerberos cache in the default -# location because of use of UID in the cache location: -# default_ccache_name = KEYRING:persistent:%{uid} -# -# kinit: Invalid UID in persistent keyring name while getting default ccache -# -# Workaround by commenting that line from the config: -# https://community.hortonworks.com/content/supportkb/222432/error-kadminlocal-invalid-uid-in-persistent-keyrin.html -RUN sed -i 's/^ default_ccache_name/# default_ccache_name/' /etc/krb5.conf - -# Call fedora-messaging CLI and tell it to use the ComposeTracker -# class from the compose-tracker module. -CMD fedora-messaging --conf /work/my_config.toml consume --callback=coreos_koji_tagger:Consumer - -# Put the keytab in place -ADD coreosbot.keytab /work/ diff --git a/README.md b/README.md index 3c264e7..55360df 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,2 @@ -# coreos-koji-tagger -Source code that monitors a git repo and tags packages over into -appropriate koji tags to be consumed by Fedora CoreOS build processes. - -# Rough notes for deployment - -Create a new project and build the container. - -``` -oc new-project coreos-koji-tagger -oc new-build --strategy=docker https://pagure.io/dusty/coreos-koji-tagger --to coreos-koji-tagger-img -``` - -Use kedge to get up and running in openshift: - -``` -kedge apply -f kedge.yaml -``` +moved to https://github.com/coreos/fedora-coreos-koji-tagger diff --git a/coreos_koji_tagger.py b/coreos_koji_tagger.py deleted file mode 100755 index 4c36b0d..0000000 --- a/coreos_koji_tagger.py +++ /dev/null @@ -1,499 +0,0 @@ -#!/usr/bin/python3 -import fedora_messaging.api -import os -import re -import requests -import logging -import json - -import dnf.subject -import hawkey - -import sys -import subprocess -import requests - -# Set local logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -# The target and the intermediate tag. The target tag is where we want -# builds to end up. We'll check the target tag to see if builds are already -# there. The intermediate tag is used when tagging. It is useful to -# set the intermediate tag different than the target tag when there is -# an intermediate tag that is set up for signing. For example we use -# f{releasever}-signing-pending tags today. They inherit from the coreos-pool -# and are configured to sign rpms and then move them into the -# coreos-pool tag. -KOJI_TARGET_TAG = 'coreos-pool' -KOJI_INTERMEDIATE_TAG = 'f{releasever}-coreos-signing-pending' - -# if we are in a stage environment then use the -# stage koji as well as the staging kerberos -if os.getenv('COREOS_KOJI_TAGGER_USE_STG', 'false') == 'true': - KOJI_CMD = '/usr/bin/stg-koji' - KERBEROS_DOMAIN = 'STG.FEDORAPROJECT.ORG' -else: - KOJI_CMD = '/usr/bin/koji' - KERBEROS_DOMAIN = 'FEDORAPROJECT.ORG' - -# The name of the FAS user account associated with the keytab -KOJI_COREOS_USER = 'coreosbot' - -GIT_REPO_DOMAIN = 'https://pagure.io/' -GIT_REPO_FULLNAME = 'dusty/coreos-koji-data' -GIT_REPO_BRANCH = 'master' - -# We are processing the io.pagure.prod.pagure.git.receive topic -# https://apps.fedoraproject.org/datagrepper/raw?topic=io.pagure.prod.pagure.git.receive&delta=100000 -EXAMPLE_MESSAGE_BODY = json.loads(""" -{ - "forced": false, - "agent": "dustymabe", - "repo": { - "custom_keys": [], - "description": "coreos-koji-data", - "parent": null, - "date_modified": "1558714988", - "access_users": { - "admin": [], - "commit": [], - "ticket": [], - "owner": [ - "dustymabe" - ] - }, - "namespace": "dusty", - "priorities": {}, - "id": 6234, - "access_groups": { - "admin": [], - "commit": [], - "ticket": [] - }, - "milestones": {}, - "user": { - "fullname": "Dusty Mabe", - "name": "dustymabe" - }, - "date_created": "1558714988", - "fullname": "dusty/coreos-koji-data", - "url_path": "dusty/coreos-koji-data", - "close_status": [], - "tags": [], - "name": "coreos-koji-data" - }, - "end_commit": "db5c806769a5ab35bfeb15e1ac7c727ec1275b23", - "branch": "master", - "authors": [ - { - "fullname": "Dusty Mabe", - "name": "dustymabe" - } - ], - "total_commits": 1, - "start_commit": "db5c806769a5ab35bfeb15e1ac7c727ec1275b23" -} -""" -) - - -# Given a repo (and thus an input JSON) analyze existing koji tag set -# and tag in any missing packages - -class Consumer(object): - def __init__(self): - self.target_tag = KOJI_TARGET_TAG - self.intermediate_tag = KOJI_INTERMEDIATE_TAG - self.koji_user = KOJI_COREOS_USER - self.kerberos_domain = KERBEROS_DOMAIN - self.git_repo_domain = GIT_REPO_DOMAIN - self.git_repo_fullname = GIT_REPO_FULLNAME - self.git_repo_branch = GIT_REPO_BRANCH - - # If a keytab was specified let's use it - self.keytab_file = os.environ.get('COREOS_KOJI_TAGGER_KEYTAB_FILE') - if self.keytab_file: - if os.path.exists(self.keytab_file): - self.kinit() - else: - raise - else: - logger.info('No keytab file defined in ' - '$COREOS_KOJI_TAGGER_KEYTAB_FILE') - logger.info('Will not attempt koji write operations') - - def __call__(self, message: fedora_messaging.api.Message): - logger.debug(message.topic) - logger.debug(message.body) - - # Re-attempt to kinit if our authentication has timed out - if self.keytab_file: - if check_koji_connection().returncode != 0: - self.kinit() - - # Grab the raw message body and the status from that - msg = message.body - branch = msg['branch'] - repo = msg['repo']['fullname'] - commit = msg['end_commit'] - - if (repo != self.git_repo_fullname): - logger.debug(f'Skipping message from unrelated repo: {repo}') - return - - if (branch != self.git_repo_branch): - logger.info(f'Skipping message from unrelated branch: {branch}') - return - - # Now grab data from the commit we should operate on: - # https://pagure.io/dusty/coreos-koji-data/raw/db5c806769a5ab35bfeb15e1ac7c727ec1275b23/f/data.json - # This data file is basically a list ['build1NVR', 'build2NVR', 'etc'] - url = f'{self.git_repo_domain}/{self.git_repo_fullname}/raw/{commit}/f/data.json' - logger.info(f'Attempting to retrieve data from {url}') - r = requests.get(url) - - - # NOMENCLATURE: - # - # In koji there is the concept of a pkg and a build. A pkg - # is a piece of software (i.e. kernel) whereas a build is a - # specific build of that software that is unique by NVR (i.e. - # kernel-5.0.17-300.fc30). RPMs are output of a build. There - # can be many rpms (including subpackages) output from a build - # (for example kernel-5.0.17-300.fc30.x86_64.rpm and - # kernel-devel-5.0.17-300.fc30.x86_64.rpm). So we have: - # - # kernel --> koji pkg - # kernel-5.0.17-300.fc30 --> koji build (matches srpm name) - # kernel-5.0.17-300.fc30.x86_64 --> main rpm package - # kernel-devel-5.0.17-300.fc30.x86_64 --> rpm subpackage - # - # STRATEGY: - # - # The lockfile input gives a list of rpm names in NEVRA format. We - # must derive the srpm name (koji build name) from that and compare - # that with existing koji builds in the tag. Once we have a list of - # koji builds that aren't in the tag we can add the koji pkg to the - # tag (if needed) and then tag the koji build into the tag. - - # parse the lockfile and get a set of rpm NEVRAs - desiredrpms = set(parse_lockfile_data(r.text)) - - # convert the rpm NEVRAs into a list of srpm NVRA (format of koji - # build name) - buildsinfo = get_buildsinfo_from_rpmnevras(desiredrpms) - desiredbuilds = set(buildsinfo.keys()) - - # Grab the list of pkgs that can be tagged into the tag - pkgsintag = get_pkgs_in_tag(self.target_tag) - - # Grab the currently tagged builds and convert it into a set - currentbuilds = set(get_tagged_builds(self.target_tag)) - - # Find out the difference between the current set of builds - # that exist in the koji tag and the desired set of builds to - # be added to the koji tag. - buildstotag = list(desiredbuilds.difference(currentbuilds)) - - - # compute the package names of each build and determine whether - # it is in the tag or not. If not we'll need to add the package - # to the tag before we can add the specific build to the tag - pkgstoadd = [] - for build in buildstotag: - - # Find the some defining information for this build. - buildinfo = get_rich_info_for_rpm_string(build, arch=False) - - # Check to see if the koji pkg is already covered by the tag - if buildinfo.name not in pkgsintag: - pkgstoadd.append(buildinfo.name) - - # Log if there is nothing to do - if not pkgstoadd and not buildstotag: - logger.info(f'No new builds to tag.. going back to sleep') - return - - # Add the needed packages to the tag if we have credentials - if pkgstoadd: - logger.info('Adding packages to the ' - f'{self.target_tag} tag: {pkgstoadd}') - if self.keytab_file: - add_pkgs_to_tag(tag=self.target_tag, - pkgs=pkgstoadd, - owner=self.koji_user) - logger.info('Package adding done') - - # Perform the tagging for each release into the intermediate - # tag for that release if we have credentials - if buildstotag: - releasevers = set(buildsinfo.values()) - for releasever in releasevers: - tag = self.intermediate_tag.format(releasever=releasever) - buildstotagforthisrelease = \ - [x for x in buildstotag if buildsinfo[x] == releasever] - if buildstotagforthisrelease: - logger.info('Tagging builds into the ' - f'{tag} tag: {buildstotagforthisrelease}') - if self.keytab_file: - tag_builds(tag=tag, builds=buildstotagforthisrelease) - logger.info('Tagging done') - - def kinit(self): - logger.info(f'Authenticating with keytab: {self.keytab_file}') - cmd = f'/usr/bin/kinit -k -t {self.keytab_file}' - cmd += f' {self.koji_user}@{self.kerberos_domain}' - runcmd(cmd.split(' '), check=True) - check_koji_connection(check=True) # Make sure it works - -def runcmd(cmd: list, **kwargs: int) -> subprocess.CompletedProcess: - try: - logger.debug(f'Running command: {cmd}') - cp = subprocess.run(cmd, **kwargs) - except subprocess.CalledProcessError as e: - logger.error('Running command returned bad exitcode') - logger.error(f'COMMAND: {cmd}') - logger.error(f' STDOUT: {e.stdout}') - logger.error(f' STDERR: {e.stderr}') - raise - return cp # subprocess.CompletedProcess - -def get_rich_info_for_rpm_string(string: str, arch: bool) -> hawkey.NEVRA: - # arch: (bool) whether arch is included in the string - if arch: - form=hawkey.FORM_NEVRA - else: - form=hawkey.FORM_NEVR - - # get a hawkey.Subject object for the string - subject = dnf.subject.Subject(string) # returns hawkey.Subject - - # get a list of hawkey.NEVRA objects that are the possibilities - nevras = subject.get_nevra_possibilities(forms=form) - - # return the first hawkey.NEVRA item in the list of possibilities - info = nevras[0] - # print(info.name) - # print(info.version) - # print(info.epoch) - # print(info.release) - # print(info.arch) - return info - -def parse_lockfile_data(text: str) -> list: - # Parse the rpm lockfile format and return a list of rpms in - # NEVRA form. - # Best documention on the format for now: - # https://github.com/projectatomic/rpm-ostree/commit/8ff0ee9c89ecc0540182b5b506455fc275d27a61 - # - # An example looks something like: - # - # { - # "packages" : [ - # [ - # "GeoIP-1.6.12-5.fc30.x86_64", - # "sha256:21dc1220cfdacd089c8c8ed9985801a9d09edb7c26543694cef57ada1d8aafa8" - # ], - # } - - # The data is JSON (yay) - data = json.loads(text) - logger.debug('Retrieved JSON data:') - logger.debug(json.dumps(data, indent=4, sort_keys=True)) - - # The data structure is a list of lists where the first item - # in each sublist is the package name in NEVRA format. We only - # care about the NEVRA so just grab that and return it. - packages = [x[0] for x in data['packages']] - return packages - -def grab_first_column(text: str) -> list: - # The output is split by newlines (split \n) and contains an - # extra newline at the end (rstrip). We only care about the 1st - # column (split(' ')[0]) so just grab that and return a list. - lines = text.rstrip().split('\n') - return [b.split(' ')[0] for b in lines] - -def get_releasever_from_buildroottag(buildroottag: str) -> str: - logger.debug(f'Checking buildroottag {buildroottag}') - if 'afterburn' in buildroottag: - # example: module-afterburn-rolling-3020190524194016-2c789dff-build - releasever = re.search('module-afterburn-rolling-(\d\d)', - buildroottag).group(1) - else: - # example: f30-build - releasever = re.search('f(\d\d)', buildroottag).group(1) - if not releasever: - raise - return releasever - -def get_buildsinfo_from_rpmnevras(rpmnevras: set) -> dict: - # Given a list of rpm NEVRAs get the list of srpms (and - # thus koji build names) from it - if not rpmnevras: - raise - - # Get a set of NVRAs from the list of NEVRAs - rpmnvras = set() - for rpmnevra in rpmnevras: - # Find the some defining information for this rpm. - rpminfo = get_rich_info_for_rpm_string(rpmnevra, arch=True) - # come up with rpm NVRA - rpmnvra = f"{rpminfo.name}-{rpminfo.version}-{rpminfo.release}.{rpminfo.arch}" - rpmnvras.add(rpmnvra) - - # Query koji in a single query to get rpminfo (includes SRPM name - # and buildroot tag name) for all rpmnvras - # - # Usage: koji rpminfo [options] [ ...] - cmd = f'/usr/bin/koji rpminfo'.split(' ') - cmd+= rpmnvras - cp = runcmd(cmd, check=True, capture_output=True, text=True) - - # Outputs formatting like: - # - `SRPM: E:N-V-R` - # - `Buildroot: 16295881 (tag f30-build, arch x86_64, repo 1166504) - # - # $ koji rpminfo grub2-efi-x64-2.02-81.fc30.x86_64 - # RPM: 1:grub2-efi-x64-2.02-81.fc30.x86_64 [17584661] - # RPM Path: /mnt/koji/packages/grub2/2.02/81.fc30/x86_64/grub2-efi-x64-2.02-81.fc30.x86_64.rpm - # SRPM: 1:grub2-2.02-81.fc30 [1269330] - # SRPM Path: /mnt/koji/packages/grub2/2.02/81.fc30/src/grub2-2.02-81.fc30.src.rpm - # Built: Mon, 20 May 2019 13:19:34 EDT - # SIGMD5: bbfb797611097256c119f99c4480e5a8 - # Size: 365984 - # License: GPLv3+ - # Build ID: 1269330 - # Buildroot: 16295881 (tag f30-build, arch x86_64, repo 1166504) - # Build Host: bkernel03.phx2.fedoraproject.org - # Build Task: 34957405 - - # Go through each line and get the srpm names - srpm = None - buildroottag = None - buildsinfo = dict() - for line in cp.stdout.strip().splitlines(): - if 'SRPM:' in line: - # The (\d+:)? pulls the epoch off the front of each SRPM value - # if it exists. - srpm = re.search('SRPM: (\d+:)?([\S]+)', line).group(2) - if 'Buildroot:' in line: - buildroottag = re.search('Buildroot: [\d]+ \(tag ([\S]+),', line).group(1) - releasever = get_releasever_from_buildroottag(buildroottag) - # Now that we have both pieces of info we add to the dict - buildsinfo.update({srpm: releasever}) - srpm = None - buildroottag = None - - logger.debug("Found Builds: {}".format(buildsinfo.keys())) - logger.debug(buildsinfo) - return buildsinfo - -def get_tagged_builds(tag: str) -> list: - if not tag: - raise - - # Grab current builds in the koji tag - # The output with `--quiet` is like this: - # - # coreos-installer-0-5.gitd3fc540.fc30 coreos-pool dustymabe - # ignition-2.0.0-beta.3.git910e6c6.fc30 coreos-pool jlebon - # kernel-5.0.10-300.fc30 coreos-pool labbott - # kernel-5.0.11-300.fc30 coreos-pool labbott - # - # Usage: koji list-tagged [options] tag [package] - cmd = f'/usr/bin/koji list-tagged {tag} --quiet'.split(' ') - cp = runcmd(cmd, check=True, capture_output=True, text=True) - return grab_first_column(cp.stdout) - -def get_pkgs_in_tag(tag: str) -> list: - if not tag: - raise - # Usage: koji list-pkgs [options] - cmd = f'/usr/bin/koji list-pkgs --tag={tag} --quiet'.split(' ') - cp = runcmd(cmd, check=True, capture_output=True, text=True) - return grab_first_column(cp.stdout) - -def tag_builds(tag: str, builds: list): - if not tag or not builds: - raise - # Usage: koji tag-build [options] [...] - cmd = f'/usr/bin/koji tag-build {tag}'.split(' ') - cmd.extend(builds) - runcmd(cmd, check=True) - -def add_pkgs_to_tag(tag: str, pkgs: list, owner: str): - if not tag or not pkgs or not owner: - raise - # Usage: koji add-pkg [options] tag package [package2 ...] - cmd = f'/usr/bin/koji add-pkg {tag} --owner {owner}'.split(' ') - cmd.extend(pkgs) - runcmd(cmd, check=True) - -def check_koji_connection(check: bool = False) -> subprocess.CompletedProcess: - # Usage: koji moshimoshi [options] - cmd = f'/usr/bin/koji moshimoshi'.split(' ') - cp = runcmd(cmd, check=check, capture_output=True) - return cp - -# The code in this file is expected to be run through fedora messaging -# However, you can run the script directly for testing purposes. The -# below code allows us to do that and also fake feeding data to the -# call by updating the json text below. -if __name__ == '__main__': - sh = logging.StreamHandler() - sh.setFormatter(logging.Formatter('%(asctime)s %(message)s')) - logger.addHandler(sh) - - # Mock the web request to get the data so that we can easily - # modify the below values in order to run a test: - from unittest.mock import Mock - - requests_response = Mock() - requests_response.text = """ -{ - "packages" : [ - [ - "GeoIP-1.6.12-5.fc30.x86_64", - "sha256:21dc1220cfdacd089c8c8ed9985801a9d09edb7c26543694cef57ada1d8aafa8" - ], - [ - "GeoIP-GeoLite-data-2018.06-3.fc30.noarch", - "sha256:b871f757d061af1125280219dca15b5066018b6ff20c08010c5774c484f127a8" - ], - [ - "NetworkManager-1:1.16.2-1.fc30.x86_64", - "sha256:4818f336e9496ba919dd8158172d57b77cb65389f4b6c0d2462fea3a29ad9fda" - ], - [ - "NetworkManager-libnm-1:1.16.2-1.fc30.x86_64", - "sha256:f973761517dd7fd2dcfff0aa9578c99e509591a87d0fc316751c0f96e045cbc1" - ], - [ - "acl-2.2.53-3.fc30.x86_64", - "sha256:af5d6641b71ec62d126fa71322a8451aa3de7948202633da802bccd1fd6ece45" - ], - [ - "adcli-0.8.2-3.fc30.x86_64", - "sha256:ff7862e1b1fefe936f3ae614d008835e686ccdc6ec06e7cf445e8f75d73d50f0" - ], - [ - "linux-atm-libs-2.5.1-21.fc29.x86_64", - "sha256:7dfd2156bd09e02a93a54eedea533b44afc41d06df28f2add364e5327b27c0f7" - ], - [ - "afterburn-4.1.0-2.module_f30+4375+41ed41a6.x86_64", - "sha256:c1d3aa735ea13a8831e92a54b5c0a32b33cbfb5610cfeda8294b2163dd2f5699" - ] - ] -} - """ - requests = Mock() - requests.get.return_value = requests_response - - m = fedora_messaging.api.Message( - topic = 'io.pagure.prod.pagure.git.receive', - body = EXAMPLE_MESSAGE_BODY) - c = Consumer() - c.__call__(m) diff --git a/kedge.yaml b/kedge.yaml deleted file mode 100644 index e2bba44..0000000 --- a/kedge.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: coreos-koji-tagger - -deploymentConfigs: - - containers: - - image: "" -# envFrom: -# - secretRef: -# name: pagure-token - triggers: - - type: ImageChange - imageChangeParams: - automatic: true - containerNames: - - coreos-koji-tagger - from: - kind: ImageStreamTag - name: coreos-koji-tagger-img:latest -#secrets: -# - name: pagure-token -# data: -# PAGURE_TOKEN: [[ PAGURE_TOKEN ]]