From 4eaaf3723bbc87e9dcba6ae6cfd168ffc7eaef1b Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Jul 11 2016 07:19:34 +0000 Subject: Port pungi-trigger to execute pungi-signed repo rather than pungi-createrepo * get tag for related treevariant from PDC * get triggered based on rida ready state: fedmsg-logger --modname rida --topic module.state.change --message '{"state": "ready", "module": "testmodule", "version": "1.0"}' * just a note that you actually require latest hawkey for pungi to be working Signed-off-by: Lubos Kocman --- diff --git a/fedmsg.d/pungi-trigger.py b/fedmsg.d/pungi-trigger.py new file mode 100644 index 0000000..f92166d --- /dev/null +++ b/fedmsg.d/pungi-trigger.py @@ -0,0 +1,15 @@ +import socket +hostname = socket.gethostname().split('.')[0] + +config = { + # Just enough fedmsg config to start publishing... + "endpoints": { + "rida": [ + "tcp://127.0.0.1:4001" + ], + "fedora-infrastructure": [ + "tcp://stg.fedoraproject.org:9940" + ] + }, +} + diff --git a/pungi-trigger.py b/pungi-trigger.py index 0a1f96d..e0e01cc 100755 --- a/pungi-trigger.py +++ b/pungi-trigger.py @@ -5,26 +5,23 @@ kicks off pungi runs in response to module metadata changes. """ from __future__ import print_function +import json import os import sys import subprocess as sp -import tempfile -import shutil - -import yaml import fedmsg import fedmsg.config import fedmsg.meta +import pdc_client config = fedmsg.config.load_config() fedmsg.meta.make_processors(**config) -distgit_url = 'http://pkgs.stg.fedoraproject.org' -target = 'org.fedoraproject.stg.git.receive' -pubdir = os.environ['PUBDIR'] -puburl = os.environ['PUBURL'] +target = 'org.fedoraproject.dev.rida.module.state.change' +pdcurl = os.environ['PDCURL'] +repodir = os.environ['REPODIR'] def log(*args): @@ -55,15 +52,6 @@ def run(cmd): return lines - -def get_static_manifest_location(lines): - telltale = "Wrote rpm static manifest into" - for line in lines: - if telltale in line: - return line.split(telltale)[-1].strip() - raise ValueError("Could not find %r in output." % telltale) - - def get_built_repo_location(lines): telltale = "as destination dir" for line in lines: @@ -71,122 +59,54 @@ def get_built_repo_location(lines): return line.split(telltale)[0].strip().split()[-1] raise ValueError("Could not find %r in output." % telltale) -def publish_directory(path, link=None): - log(" Publishing directory:", path) - tail = path.split('/')[-1] - dest = os.path.join(pubdir, tail) - - log(" Moving content to", dest) - if os.path.exists(dest): - log(" ...first removing old content at", dest) - shutil.rmtree(dest) - shutil.move(path, dest) - - if link: - link = os.path.join(pubdir, link) - log(" Managing link at ", link) - try: - os.remove(link) - except OSError: - pass - os.symlink(dest, link) - - -def is_modularity_event(msg, config): - # XXX - HACK - ideally, we would check a 'namespace' field in the git - # message here, but we don't have such a field yet. - # See -- https://fedorahosted.org/fedora-infrastructure/ticket/5245 - - # Instead, for now, just operate on a hardcoded set. - known_modules = set([ - "testmodule", - "core", - ]) - - modules = fedmsg.meta.msg2packages(msg, **config) - return bool(set.intersection(modules, known_modules)) - +def get_tag_from_pdc(pdc_proxy, variant_name, variant_version): + tv = pdc_proxy.treevariant(page_size=-1, variant_id=variant_name, variant_version=variant_version) + log ("Pdc returned %s" % (str(tv))) + if not tv: + return None + if len(tv) != 1: + log("Received multiple tree variants=%s, expected exactly 1" % tv) + return None + return tv[0]['koji_tag'] def main(): + pdc_proxy = pdc_client.PDCClient(pdcurl, insecure=True, develop=True) + log("* Listening for messages on topic %r" % target) + fedmsg.crypto.init(**config) for _, _, topic, msg in fedmsg.tail_messages(topic=target, **config): + print ("Handling msg %s" % fedmsg.encoding.pretty_dumps(msg)) idx = msg['msg_id'] log("Saw a message", topic, idx) - - if not is_modularity_event(msg, config): - # Throw it away - log("Dropping", topic, idx) + try: + module = json.loads(msg['msg'][msg['msg'].keys()[0]]) # get msg body + except ValueError, e: + log(e) + continue + if module['state'] != "ready": continue + # Run signed repo log("Working with", topic, idx) + sigkeys = " ".join(['unsigned', ]) # XXX: fix this + tag = get_tag_from_pdc(pdc_proxy, variant_name=module['module'], variant_version=module['version']) + if not tag: + log("Couldn't get tag for module %s-%s" % (module['module'], module['version'])) + continue - # There should only be one package (aka, 'module') here.. - module = list(fedmsg.meta.msg2packages(msg, **config))[0] - - # Create a temp space to start the work.. - checkout = tempfile.mktemp() try: - # Get the metadata that changed - link = '%s/cgit/modules/%s.git' % (distgit_url, module) - lines = run(['git', 'clone', link, checkout]) - - files = os.listdir(checkout) - files = [f for f in files if f.endswith('.yaml')] - - for filename in files: - # Run depsolver - lines = run([ - 'pungi-gather', - '--arch', 'x86_64', - '--config', checkout + '/' + filename, - '--target-dir', os.environ['TARGET_DIR'], - '--source-repo-from-path', os.environ['SOURCE_REPO'], - ]) - - # Run linker/createrepo with static manifest - manifest = get_static_manifest_location(lines) - log(" Got this manifest for pungi-createrepo", manifest) - lines = run([ - 'pungi-createrepo', - '--target-dir', os.environ['TARGET_DIR'], - '--source-repo-from-path', os.environ['SOURCE_REPO'], - '--static-content-manifest', manifest, - '--arch', 'x86_64', - '--extra-file', 'modulemd.yaml,' + manifest + '/fm-metadata.yaml' - ]) - - # Lastly, move the built module into a web accessible location. - repopath = get_built_repo_location(lines) - log(" Got this repopath for web publication", repopath) - - publish_directory(manifest) - publish_directory(repopath, link=module) - - # And, final step, update the modules.yaml index with a note. - # Read it - indexfile = os.path.join(pubdir, 'modules.yaml') - log(" Modifying indexfile at", indexfile) - try: - with open(indexfile, 'rb') as f: - index = yaml.load(f.read().decode('utf-8')) - index = index or {} # Protect against 'None' - except IOError: - index = {} - - # Modify it - suffix = 'rpms/repodata/modulemd.yaml' - index[module.decode('utf-8')] = \ - os.path.join(puburl, module, suffix).decode('utf-8') - - # And write it back. - with open(indexfile, 'wb') as f: - f.write(yaml.dump(index).encode('utf-8')) - finally: - try: - shutil.rmtree(checkout) - except: - pass - + log(" Running signed-repo for tag=%s", tag) + cmd = [ + 'pungi-signed-repo-prototype', + '--target-dir', os.environ['REPODIR'], + '--koji-profile', os.environ['KOJI_PROFILE'], + tag, + sigkeys, + ] + lines = run(cmd) + log("$ %s returned '%s'" % (" ".join(cmd), "\n".join( lines))) + except OSError: + log("Execution of $ %s failed" % " ".join(cmd)) if __name__ == '__main__': sanity_check() diff --git a/systemd/pungi-modularity.env b/systemd/pungi-modularity.env index 8adfc09..2fdbddf 100644 --- a/systemd/pungi-modularity.env +++ b/systemd/pungi-modularity.env @@ -1,8 +1,9 @@ # Just an example env file. In prod/stg, we'll use some real values here. # See https://pagure.io/pungi-modularity/ -export MODULARITY_DIR=/home/threebean/devel/pungi-modularity/modularity -export PYTHONPATH=$MODULARITY_DIR/productmd:$MODULARITY_DIR/pungi:$MODULARITY_DIR/pungi-gather:$MODULARITY_DIR/fm-metadata +export PYTHONPATH="/home/fedora/lkocman/Workspace/pungi-prototype:/home/fedora/lkocman/Workspace/libhif/build/python/hawkey:/home/fedora/lkocman/Workspace/dnf:/home/fedora/lkocman/Workspace/modulemd:/home/fedora/lkocman/Workspace/PyYAML-3.11/lib:/home/fedora/lkocman/Workspace/pdc-client:/home/fedora/lkocman/Workspace/beanbag" +export LD_LIBRARY_PATH="/home/fedora/lkocman/Workspace/libhif/build/libhif" export PATH=$MODULARITY_DIR/pungi-gather/bin:$MODULARITY_DIR/pungi/bin:$PATH -export TARGET_DIR=~/RHWorkdir -export SOURCE_REPO=fedora-24-alpha-x86_64 +export REPODIR=/home/fedora/lkocman +export PDCURL=http://fed-mod.org:8000/rest_api/v1/ +export KOJI_PROFILE="koji"