From 7d2c8c677a4ef7b85acc7ef01499c6a6326994da Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Aug 05 2020 16:04:04 +0000 Subject: [PATCH 1/2] Changes to packaging Signed-off-by: Mohan Boddu --- diff --git a/fedora-minimal-compose.py b/fedora-minimal-compose.py deleted file mode 100755 index 07b8da0..0000000 --- a/fedora-minimal-compose.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/python3 - -from odcs.client.odcs import ODCS, AuthMech, ComposeSourceRawConfig -import fedora_messaging.api -import fedora_messaging.config -import koji -import logging -import os - -# Set local logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -fedora_messaging.config.conf.load_config("/etc/fedora-messaging/fedora-minimal-compose.toml") - - -class Consumer(object): - """ - A fedora-messaging consumer that process messages - """ - - def __init__(self): - """ - Configure the consumer - """ - self.config = fedora_messaging.config.conf["consumer_config"] - self.rawhide = self.config['rawhide'] - self.fmc_tag = f'f{self.rawhide}-minimal-compose' - self.odcs_url = self.config['odcs_url'] - self.odcs_file = self.config['odcs_file'] - self.packages_list = self.config['packages'] - # ['anaconda', 'authconfig', 'python-blivet', 'pyparted', 'parted', 'pykickstart', 'blivet-gui', 'libblockdev', 'e2fsprogs', 'dosfstools', 'grub2', 'shim-signed', 'libselinux'] - # When we have the ability to run one compose for multiple builds, use the following list - # packages_list = ['anaconda', 'authconfig', 'python-blivet', 'pyparted', 'parted', 'pykickstart', 'blivet-gui', 'libblockdev', 'e2fsprogs', 'dosfstools', 'grub2', 'shim-signed', 'libselinux', 'chrony', 'fcoe-utils', 'hfsplus-tools', 'firewalld', 'realmd', 'yelp', 'libtimezonemap', 'kernel', 'systemd', 'dracut', 'plymouth', 'gtk3', 'python3'] - # Create a koji session - self.kojisession = koji.ClientSession(self.config['koji_url']) - self.kojisession.gssapi_login(principal=self.config['principal'], keytab=self.config['keytab']) - self.kojisession.multicall = True - - def save_nvr_to_file(self, nvr): - with open('builds_compose.log', 'w+') as f: - logger.info(f'Build that triggered the compose:{nvr}') - f.write(nvr + '\n') - - def run_koji_tasks(self, new_build): - # koji untag current builds - old_builds = self.kojisession.listTagged(self.fmc_tag) - logger.info(f'Untagging old builds from {self.fmc_tag} : {old_builds}') - for old_build in old_builds: - try: - self.kojisession.untagBuild(self.fmc_tag, old_build) - except Exception as e: - logger.error(f'Untagging {old_build} from {self.fmc_tag} failed, exception: {e}') - self.kojisession.multiCall() - # tag the new build - logger.info(f'Tagging the koji build {new_build} to {self.fmc_tag} tag') - try: - self.kojisession.tagBuild(self.fmc_tag, new_build) - except Exception as e: - logger.error(f'Tagging {new_build} from {self.fmc_tag} failed, exception: {e}') - self.kojisession.multiCall() - - def get_odcs_token(self, odcs_file): - if os.path.exists(odcs_file): - f = open(odcs_file, "r") - token = f.readline() - return token - else: - logger.error('ODCS token file is missing.') - - def run_compose(self, odcs_token): - odcs = ODCS(self.odcs_url, - auth_mech=AuthMech.OpenIDC, - openidc_token=odcs_token) - source = ComposeSourceRawConfig("releng_fmc", "master") - - compose_info = odcs.request_compose(source) - logger.info(compose_info) - return compose_info - - def inject_builds_compose(self, compose_info): - compose_id = compose_info.get('id') - os.rename('builds_compose.log', f'/srv/odcs/odcs-{compose_id}/logs/builds_compose.log') - - def __call__(self, message: fedora_messaging.api.Message): - """ - Invoked when a message is received by the consumer. - """ - logger.debug(message.topic) - logger.debug(message.body) - build = message.body['build']['nvr'] - package = message.body['build']['name'] - # Check if the package is the one of the packages we are interested in - if package in self.packages_list: - # Check if its rawhide only build - # TODO: Extend it to branched. - # Get the dist tag of the build - dist_tag = build[-4:] - - if f'fc{self.rawhide}' in dist_tag: - self.save_nvr_to_file(build) - # Pre compose koji tasks - self.run_koji_tasks(build) - # Get ODCS token - odcs_token = self.get_odcs_token(self.odcs_file) - # Run the compose - compose_info = self.run_compose(odcs_token) - # Inject the file to into the compose log dir - self.inject_builds_compose(compose_info) diff --git a/fedora-minimal-compose.spec b/fedora-minimal-compose.spec deleted file mode 100644 index 29c12ac..0000000 --- a/fedora-minimal-compose.spec +++ /dev/null @@ -1,40 +0,0 @@ -Name: fmc -Version: 0.0.0 -Release: 1%{?dist} -Summary: Fedora Minimal COmpose - -License: GPLv2+ -URL: https://pagure.io/%{name} -Source0: https://releases.pagure.org/%{name}/%{name}-%{version}.tar.bz2 - -BuildArch: noarch - -BuildRequires: python3-setuptools - -BuildRequires: python3-devel - -Requires: koji -Requires: python3-fedora-messaging -Requires: python3-ogr -Requires: fedora-messaging -Requires: python3-odcs-client - -%description -Fedoma minimal compose - -%prep -%autosetup -n %{name} - -%build - -%install -mkdir -p %{buildroot}%{_bindir}/ -install -m 755 fedora-minimal-compose.py %{buildroot}%{_bindir}/ - -%files -%{_bindir}/* - -%changelog -* Tue Jun 02 2020 Mohan Boddu - 0.0.0-1 -- First Build - diff --git a/fedora_minimal_compose.py b/fedora_minimal_compose.py new file mode 100755 index 0000000..07b8da0 --- /dev/null +++ b/fedora_minimal_compose.py @@ -0,0 +1,108 @@ +#!/usr/bin/python3 + +from odcs.client.odcs import ODCS, AuthMech, ComposeSourceRawConfig +import fedora_messaging.api +import fedora_messaging.config +import koji +import logging +import os + +# Set local logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +fedora_messaging.config.conf.load_config("/etc/fedora-messaging/fedora-minimal-compose.toml") + + +class Consumer(object): + """ + A fedora-messaging consumer that process messages + """ + + def __init__(self): + """ + Configure the consumer + """ + self.config = fedora_messaging.config.conf["consumer_config"] + self.rawhide = self.config['rawhide'] + self.fmc_tag = f'f{self.rawhide}-minimal-compose' + self.odcs_url = self.config['odcs_url'] + self.odcs_file = self.config['odcs_file'] + self.packages_list = self.config['packages'] + # ['anaconda', 'authconfig', 'python-blivet', 'pyparted', 'parted', 'pykickstart', 'blivet-gui', 'libblockdev', 'e2fsprogs', 'dosfstools', 'grub2', 'shim-signed', 'libselinux'] + # When we have the ability to run one compose for multiple builds, use the following list + # packages_list = ['anaconda', 'authconfig', 'python-blivet', 'pyparted', 'parted', 'pykickstart', 'blivet-gui', 'libblockdev', 'e2fsprogs', 'dosfstools', 'grub2', 'shim-signed', 'libselinux', 'chrony', 'fcoe-utils', 'hfsplus-tools', 'firewalld', 'realmd', 'yelp', 'libtimezonemap', 'kernel', 'systemd', 'dracut', 'plymouth', 'gtk3', 'python3'] + # Create a koji session + self.kojisession = koji.ClientSession(self.config['koji_url']) + self.kojisession.gssapi_login(principal=self.config['principal'], keytab=self.config['keytab']) + self.kojisession.multicall = True + + def save_nvr_to_file(self, nvr): + with open('builds_compose.log', 'w+') as f: + logger.info(f'Build that triggered the compose:{nvr}') + f.write(nvr + '\n') + + def run_koji_tasks(self, new_build): + # koji untag current builds + old_builds = self.kojisession.listTagged(self.fmc_tag) + logger.info(f'Untagging old builds from {self.fmc_tag} : {old_builds}') + for old_build in old_builds: + try: + self.kojisession.untagBuild(self.fmc_tag, old_build) + except Exception as e: + logger.error(f'Untagging {old_build} from {self.fmc_tag} failed, exception: {e}') + self.kojisession.multiCall() + # tag the new build + logger.info(f'Tagging the koji build {new_build} to {self.fmc_tag} tag') + try: + self.kojisession.tagBuild(self.fmc_tag, new_build) + except Exception as e: + logger.error(f'Tagging {new_build} from {self.fmc_tag} failed, exception: {e}') + self.kojisession.multiCall() + + def get_odcs_token(self, odcs_file): + if os.path.exists(odcs_file): + f = open(odcs_file, "r") + token = f.readline() + return token + else: + logger.error('ODCS token file is missing.') + + def run_compose(self, odcs_token): + odcs = ODCS(self.odcs_url, + auth_mech=AuthMech.OpenIDC, + openidc_token=odcs_token) + source = ComposeSourceRawConfig("releng_fmc", "master") + + compose_info = odcs.request_compose(source) + logger.info(compose_info) + return compose_info + + def inject_builds_compose(self, compose_info): + compose_id = compose_info.get('id') + os.rename('builds_compose.log', f'/srv/odcs/odcs-{compose_id}/logs/builds_compose.log') + + def __call__(self, message: fedora_messaging.api.Message): + """ + Invoked when a message is received by the consumer. + """ + logger.debug(message.topic) + logger.debug(message.body) + build = message.body['build']['nvr'] + package = message.body['build']['name'] + # Check if the package is the one of the packages we are interested in + if package in self.packages_list: + # Check if its rawhide only build + # TODO: Extend it to branched. + # Get the dist tag of the build + dist_tag = build[-4:] + + if f'fc{self.rawhide}' in dist_tag: + self.save_nvr_to_file(build) + # Pre compose koji tasks + self.run_koji_tasks(build) + # Get ODCS token + odcs_token = self.get_odcs_token(self.odcs_file) + # Run the compose + compose_info = self.run_compose(odcs_token) + # Inject the file to into the compose log dir + self.inject_builds_compose(compose_info) diff --git a/fedora_minimal_compose.spec b/fedora_minimal_compose.spec new file mode 100644 index 0000000..aaf7d70 --- /dev/null +++ b/fedora_minimal_compose.spec @@ -0,0 +1,41 @@ +Name: fmc +Version: 0.0.0 +Release: 1%{?dist} +Summary: Fedora Minimal Compose + +License: GPLv2+ +URL: https://pagure.io/releng/%{name} +Source0: https://releases.pagure.org/%{name}/%{name}-%{version}.tar.bz2 + +BuildArch: noarch + +BuildRequires: python3-setuptools + +BuildRequires: python3-devel + +Requires: koji +Requires: python3-fedora-messaging +Requires: python3-ogr +Requires: fedora-messaging +Requires: python3-odcs-client + +%description +Fedora minimal compose + +%prep +%autosetup -n %{name} + +%build +%py3_build + +%install +%py3_install + +%files +%pycached %{python3_sitelib}/fedora_minimal_compose.py +%{python3_sitelib}/fedora_minimal_compose-%{version}-py%{python3_version}.egg-info + +%changelog +* Tue Jun 02 2020 Mohan Boddu - 0.0.0-1 +- First Build + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..741f218 --- /dev/null +++ b/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup + +description = \ + 'Fedora Minimal Compose' + +setup( + author='Red Hat, Inc.', + author_email='mboddu@redhat.com', + description=description, + license='GPLv2+', + name='fedora_minimal_compose', + py_modules = ['fedora_minimal_compose'], + url='https://pagure.io/releng/fmc', + version='0.0.0', +) From abdf4a0a1ccdff3e1b621a83bd3b046ed504dabd Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Aug 05 2020 17:24:51 +0000 Subject: [PATCH 2/2] Fix fedora-messaging consumer call Signed-off-by: Mohan Boddu --- diff --git a/fedora-minimal-compose.toml b/fedora-minimal-compose.toml index ed528fd..62e2bbe 100644 --- a/fedora-minimal-compose.toml +++ b/fedora-minimal-compose.toml @@ -2,7 +2,7 @@ # # This file is in the TOML format. amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" -callback = "fedora-minimal-compose:Consumer" +callback = "fedora_minimal_compose:Consumer" [tls] ca_cert = "/etc/fedora-messaging/cacert.pem"