From a43d684859540162dcfece998a5d2075704b5c94 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Mar 01 2019 16:27:04 +0000 Subject: Updates to handle including custom SRPMs for scratch module builds. Signed-off-by: Merlin Mathesius --- diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index 49eb29d..9936d7e 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -758,7 +758,11 @@ chmod 644 %buildroot/etc/rpm/macros.zz-modules raise RuntimeError("Buildroot is not prep-ed") self._koji_whitelist_packages([artifact_name]) - if '://' not in source: + + if source.startswith('cli-build/'): + # treat source as a custom srpm that has already been uploaded to koji + pass + elif '://' not in source: # treat source as an srpm and upload it serverdir = _unique_path('cli-build') callback = None diff --git a/module_build_service/manage.py b/module_build_service/manage.py index b4f7507..7e60980 100755 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -102,11 +102,12 @@ def import_module(mmd_file): @manager.option('--stream', action='store', dest="stream") @manager.option('--file', action='store', dest="yaml_file") +@manager.option('--srpms', '--srpm', nargs='*', action='store', default=[], dest="srpms") @manager.option('--skiptests', action='store_true', dest="skiptests") @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs') @manager.option('-s', '--set-stream', action='append', default=[], dest='default_streams') -def build_module_locally(local_build_nsvs=None, yaml_file=None, stream=None, skiptests=False, - default_streams=None): +def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None, + stream=None, skiptests=False, default_streams=None): """ Performs local module build using Mock """ if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']: @@ -139,6 +140,8 @@ def build_module_locally(local_build_nsvs=None, yaml_file=None, stream=None, ski for ns in default_streams: name, stream = ns.split(":") optional_params["default_streams"][name] = stream + if srpms: + optional_params["srpms"] = srpms username = getpass.getuser() if not yaml_file or not yaml_file.endswith(".yaml"): diff --git a/module_build_service/models.py b/module_build_service/models.py index fe5237e..fa09b06 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -198,6 +198,7 @@ class ModuleBuild(MBSBase): cg_build_koji_tag = db.Column(db.String) # This gets set after wait scmurl = db.Column(db.String) scratch = db.Column(db.Boolean, default=False) + # JSON encoded list of links of custom SRPMs uploaded to Koji srpms = db.Column(db.String) owner = db.Column(db.String, nullable=False) time_submitted = db.Column(db.DateTime, nullable=False) diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index d692b77..904a9e6 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -22,6 +22,7 @@ # Written by Ralph Bean # Matt Prahl # Jan Kaluza +import json import re import time import shutil @@ -162,6 +163,10 @@ def format_mmd(mmd, scmurl, module=None, session=None): if 'rpms' not in xmd['mbs']: xmd['mbs']['rpms'] = {} # Add missing data in RPM components + # TODO scrmod: Does something need to be done here for RPMs that are + # overridden by custom SRPMs in addition to what is currently being + # done in record_component_builds()? Should repository and cache + # properties be set? If so, to what? for pkgname, pkg in mmd.get_rpm_components().items(): # In case of resubmit of existing module which have been # cancelled/failed during the init state, the package @@ -312,6 +317,54 @@ def merge_included_mmd(mmd, included_mmd): # Set the modified xmd back to the modulemd mmd.set_xmd(glib.dict_values(xmd)) +def get_module_srpm_overrides(module): + """ + Make necessary preparations to use any provided custom SRPMs. + + :param module: ModuleBuild object representing the module being submitted. + :type module: :class:`models.ModuleBuild` + :return: mapping of package names to SRPM links for all packages which + have custom SRPM overrides specified + :rtype: dict[str, str] + + """ + overrides = {} + + if not module.srpms: + return overrides + + try: + # Make sure we can decode the custom SRPM list + srpms = json.loads(module.srpms) + assert isinstance(srpms, list) + except Exception: + raise ValueError("Invalid srpms list encountered: {}".format(module.srpms)) + + for source in srpms: + if source.startswith('cli-build/') and source.endswith('.src.rpm'): + # This is a custom srpm that has been uploaded to koji by rpkg + # using the package name as the basename suffixed with .src.rpm + rpm_name = os.path.basename(source)[:-len('.src.rpm')] + else: + # This should be a local custom srpm path + if not os.path.exists(source): + raise IOError("Provided srpm is missing: {}".format(source)) + # Get package name from rpm headers + try: + rpm_hdr = kobo.rpmlib.get_rpm_header(source) + rpm_name = kobo.rpmlib.get_header_field(rpm_hdr, 'name').decode('utf-8') + except Exception: + raise ValueError("Provided srpm is invalid: {}".format(source)) + + if rpm_name in overrides: + log.warning('Encountered duplicate custom SRPM "{0}"' + ' for package {1}'.format(source, rpm_name)) + continue + + log.debug('Using custom SRPM "{0}" for package {1}'.format(source, rpm_name)) + overrides[rpm_name] = source + + return overrides def record_component_builds(mmd, module, initial_batch=1, previous_buildorder=None, main_mmd=None, session=None): @@ -349,6 +402,9 @@ def record_component_builds(mmd, module, initial_batch=1, if not all_components: return + # Get map of packages that have SRPM overrides + srpm_overrides = get_module_srpm_overrides(module) + rpm_weights = module_build_service.builder.GenericBuilder.get_build_weights( [c.get_name() for c in rpm_components]) all_components.sort(key=lambda x: x.get_buildorder()) @@ -376,13 +432,20 @@ def record_component_builds(mmd, module, initial_batch=1, previous_buildorder, main_mmd, session=session) continue - component_ref = mmd.get_xmd()['mbs']['rpms'][component.get_name()]['ref'] - full_url = component.get_repository() + "?#" + component_ref + package = component.get_name() + if package in srpm_overrides: + component_ref = None + full_url = srpm_overrides[package] + log.info('Building custom SRPM "{0}"' + ' for package {1}'.format(full_url, package)) + else: + component_ref = mmd.get_xmd()['mbs']['rpms'][package]['ref'] + full_url = component.get_repository() + "?#" + component_ref # Skip the ComponentBuild if it already exists in database. This can happen # in case of module build resubmition. existing_build = models.ComponentBuild.from_component_name( - db.session, component.get_name(), module.id) + db.session, package, module.id) if existing_build: # Check that the existing build has the same most important attributes. # This should never be a problem, but it's good to be defensive here so @@ -396,12 +459,12 @@ def record_component_builds(mmd, module, initial_batch=1, build = models.ComponentBuild( module_id=module.id, - package=component.get_name(), + package=package, format="rpms", scmurl=full_url, batch=batch, ref=component_ref, - weight=rpm_weights[component.get_name()] + weight=rpm_weights[package] ) session.add(build)