From 078def2f5385d056c0f82e7588fc3d749ee6734d Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Aug 22 2016 16:26:31 +0000 Subject: Merge branch 'api_change' Signed-off-by: Nils Philippsen --- diff --git a/manage.py b/manage.py index 3862a2e..92210ae 100644 --- a/manage.py +++ b/manage.py @@ -87,40 +87,6 @@ def testpdc(): @manager.command -def testbuildroot(): - """ A helper function to test buildroot creation - """ - - # Do a locally namespaced import here to avoid importing py2-only libs in a - # py3 runtime. - from rida.builder import KojiModuleBuilder, Builder - - cfg = Config() - cfg.koji_profile = "koji" - cfg.koji_config = "/etc/rida/koji.conf" - cfg.koji_arches = ["x86_64", "i686"] - - mb = KojiModuleBuilder(module="testmodule-1.0", config=cfg) # or By using Builder - # mb = Builder(module="testmodule-1.0", backend="koji", config=cfg) - - resume = False - - if not resume: - mb.buildroot_prep() - mb.buildroot_add_dependency(["f24"]) - mb.buildroot_ready() - task_id = mb.build(artifact_name="fedora-release", - source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9") - mb.buildroot_add_artifacts(["fedora-release-24-2", ]) # just example with disttag macro - mb.buildroot_ready(artifact="fedora-release-24-2") - else: - mb.buildroot_resume() - - task_id = mb.build(artifact_name="fedora-release", - source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9") - - -@manager.command def upgradedb(): """ Upgrades the database schema to the latest revision """ diff --git a/rida/builder.py b/rida/builder.py index d86fa5d..3674622 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -89,59 +89,135 @@ KOJI_DEFAULT_GROUPS = { ] } + +""" +Example workflows - helps to see the difference in implementations +Copr workflow: + +1) create project (input: name, chroot deps: e.g. epel7) +2) optional: selects project dependencies e.g. epel-7 +3) build package a.src.rpm # package is automatically added into buildroot + after it's finished +4) createrepo (package.a.src.rpm is available) + +Koji workflow + +1) create tag, and build-tag +2) create target out of ^tag and ^build-tag +3) run regen-repo to have initial repodata (happens automatically) +4) build module-build-macros which provides "dist" macro +5) tag module-build-macro into buildroot +6) wait for module-build-macro to be available in buildroot +7) build all components from scmurl +8) (optional) wait for selected builds to be available in buildroot + +""" class GenericBuilder: - """External Api for builders""" + """ + External Api for builders + + Example usage: + config = rida.config.Config() + builder = Builder(module="testmodule-1.2-3", backend="koji", config) + builder.buildroot_connect() + builder.build(artifact_name="bash", + source="git://pkgs.stg.fedoraproject.org/rpms/bash" + "?#70fa7516b83768595a4f3280ae890a7ac957e0c7") + + ... + # E.g. on some other worker ... just resume buildroot that was initially created + builder = Builder(module="testmodule-1.2-3", backend="koji", config) + builder.buildroot_connect() + builder.build(artifact_name="not-bash", + source="git://pkgs.stg.fedoraproject.org/rpms/not-bash" + "?#70fa7516b83768595a4f3280ae890a7ac957e0c7") + # wait until this particular bash is available in the buildroot + builder.buildroot_ready(artifacts=["bash-1.23-el6"]) + builder.build(artifact_name="not-not-bash", + source="git://pkgs.stg.fedoraproject.org/rpms/not-not-bash" + "?#70fa7516b83768595a4f3280ae890a7ac957e0c7") + + """ __metaclass__ = ABCMeta backend = "generic" @abstractmethod - def buildroot_prep(self): - """ - preps buildroot + def buildroot_connect(self): """ - raise NotImplementedError() + This is an idempotent call to create or resume and validate the build + environment. .build() should immediately fail if .buildroot_connect() + wasn't called. - @abstractmethod - def buildroot_resume(self): - """ - resumes buildroot (alternative to prep) + Koji Example: create tag, targets, set build tag inheritance... """ raise NotImplementedError() @abstractmethod def buildroot_ready(self, artifacts=None): """ - :param artifacts=None : a list of artifacts supposed to be in buildroot - return when buildroot is ready (or contain specified artifact) + :param artifacts=None : a list of artifacts supposed to be in the buildroot + (['bash-123-0.el6']) + + returns when the buildroot is ready (or contains the specified artifact) + + This function is here to ensure that the buildroot (repo) is ready and + contains the listed artifacts if specified. """ raise NotImplementedError() @abstractmethod - def buildroot_add_dependency(self, dependencies): + def buildroot_add_repos(self, dependencies): """ - :param dependencies: a list off modules which we build-depend on - adds dependencies 'another module(s)' into buildroot + :param dependencies: a list of modules represented as a list of dicts, + like: + [{'name': ..., 'version': ..., 'release': ...}, ...] + + Make an additional repository available in the buildroot. This does not + necessarily have to directly install artifacts (e.g. koji), just make + them available. + + E.g. the koji implementation of the call uses PDC to get koji_tag + associated with each module dep and adds the tag to $module-build tag + inheritance. """ raise NotImplementedError() @abstractmethod def buildroot_add_artifacts(self, artifacts, install=False): """ - :param artifacts: list of artifacts to be available in buildroot - :param install=False: pre-install artifact in buildroot (otherwise "make it available for install") - add artifacts into buildroot, can be used to override buildroot macros + :param artifacts: list of artifacts to be available or installed + (install=False) in the buildroot (e.g list of $NEVRAS) + :param install=False: pre-install artifact in the buildroot (otherwise + "just make it available for install") + + Example: + + koji tag-build $module-build-tag bash-1.234-1.el6 + if install: + koji add-group-pkg $module-build-tag build bash + # This forces install of bash into buildroot and srpm-buildroot + koji add-group-pkg $module-build-tag srpm-build bash """ raise NotImplementedError() @abstractmethod def build(self, artifact_name, source): """ - :param artifact_name : a crucial, since we can't guess a valid srpm name - without having the exact buildroot (collections/macros) - used e.g. for whitelisting packages - artifact_name is used to distinguish from artifact (e.g. package x nvr) - :param source : a scmurl to repository with receipt (e.g. spec) + :param artifact_name : A package name. We can't guess it since macros + in the buildroot could affect it, (e.g. software + collections). + :param source : an SCM URL, clearly identifying the build artifact in a + repository + + The artifact_name parameter is used in koji add-pkg (and it's actually + the only reason why we need to pass it). We don't really limit source + types. The actual source is usually delivered as an SCM URL in + fedmsg['msg']['scmurl'] (currently only SCM URLs work). + + Example + .build("bash", "git://someurl/bash#damn") #build from SCM URL + .build("bash", "/path/to/srpm.src.rpm") #build from source RPM """ raise NotImplementedError() @@ -179,14 +255,14 @@ class KojiModuleBuilder(GenericBuilder): self.tag_name = tag_name self.__prep = False log.debug("Using koji profile %r" % config.koji_profile) - log.debug ("Using koji_config: %s" % config.koji_config) + log.debug("Using koji_config: %s" % config.koji_config) self.koji_session, self.koji_module = self.get_session_from_config(config) self.arches = config.koji_arches if not self.arches: raise ValueError("No koji_arches specified in the config.") - # These eventually get populated when buildroot_{prep,resume} is called + # These eventually get populated by calling _connect and __prep is set to True self.module_tag = None # string self.module_build_tag = None # string self.module_target = None # A koji target dict @@ -196,14 +272,16 @@ class KojiModuleBuilder(GenericBuilder): self.module_str, self.tag_name) @rida.utils.retry(wait_on=koji.GenericError) - def buildroot_ready(self, artifacts): - """ Returns True or False if the given artifacts are in the build root. + def buildroot_ready(self, artifacts=None): + """ + :param artifacts=None - list of nvrs + Returns True or False if the given artifacts are in the build root. """ assert self.module_target, "Invalid build target" tag_id = self.module_target['build_tag'] repo = self.koji_session.getRepo(tag_id) - builds = [self.koji_session.getBuild(a) for a in artifacts] + builds = [self.koji_session.getBuild(a) for a in artifacts or []] log.info("%r checking buildroot readiness for " "repo: %r, tag_id: %r, artifacts: %r, builds: %r" % ( self, repo, tag_id, artifacts, builds)) @@ -323,38 +401,18 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules raise ValueError("Unrecognized koji authtype %r" % authtype) return (koji_session, koji_module) - def buildroot_resume(self): # XXX: experimental - """ - Resume existing buildroot. Sets __prep=True - """ - log.info("%r resuming buildroot." % self) - chktag = self.koji_session.getTag(self.tag_name) - if not chktag: - raise SystemError("Tag %s doesn't exist" % self.tag_name) - chkbuildtag = self.koji_session.getTag(self.tag_name + "-build") - if not chkbuildtag: - raise SystemError("Build Tag %s doesn't exist" % self.tag_name + "-build") - chktarget = self.koji_session.getBuildTarget(self.tag_name) - if not chktarget: - raise SystemError("Target %s doesn't exist" % self.tag_name) - self.module_tag = chktag - self.module_build_tag = chkbuildtag - self.module_target = chktarget - self.__prep = True - log.info("%r buildroot resumed." % self) + def buildroot_connect(self): + log.info("%r connecting buildroot." % self) - def buildroot_prep(self): - """ - :param module_deps_tags: a tag names of our build requires - :param module_deps_tags: a tag names of our build requires - """ - log.info("%r preparing buildroot." % self) + # Create or update individual tags self.module_tag = self._koji_create_tag( self.tag_name, self.arches, perm="admin") # the main tag needs arches so pungi can dump it + self.module_build_tag = self._koji_create_tag( self.tag_name + "-build", self.arches, perm="admin") - groups = KOJI_DEFAULT_GROUPS # TODO: read from config + # TODO: handle in buildroot_add_artifact(install=true) and track groups as module buildrequires + groups = KOJI_DEFAULT_GROUPS if groups: @rida.utils.retry(wait_on=SysCallError, interval=5) def add_groups(): @@ -366,17 +424,19 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules self.module_target = self._koji_add_target(self.tag_name, self.module_build_tag, self.module_tag) self.__prep = True - log.info("%r buildroot prepared." % self) + log.info("%r buildroot sucessfully connected." % self) - def buildroot_add_dependency(self, dependencies): + def buildroot_add_repos(self, dependencies): tags = [self._get_tag(d)['name'] for d in dependencies] - log.info("%r adding deps for %r" % (self, tags)) + log.info("%r adding deps on %r" % (self, tags)) self._koji_add_many_tag_inheritance(self.module_build_tag, tags) def buildroot_add_artifacts(self, artifacts, install=False): """ :param artifacts - list of artifacts to add to buildroot :param install=False - force install artifact (if it's not dragged in as dependency) + + This method is safe to call multiple times. """ log.info("%r adding artifacts %r" % (self, artifacts)) dest_tag = self._get_tag(self.module_build_tag)['id'] @@ -412,32 +472,69 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules return get_result() + def _get_task_by_artifact(self, artifact_name): + """ + :param artifact_name: e.g. bash + + Searches for a tagged package inside module tag. + + Returns task_id or None. + + TODO: handle builds with skip_tag (not tagged at all) + """ + # yaml file can hold only one reference to a package name, so + # I expect that we can have only one build of package within single module + # Rules for searching: + # * latest: True so I can return only single task_id. + # * we do want only build explicitly tagged in the module tag (inherit: False) + + opts = {'latest': True, 'package': artifact_name, 'inherit': False} + tagged = self.koji_session.listTagged(self.module_tag['name'], **opts) + + if tagged: + assert len(tagged) == 1, "Expected exactly one item in list. Got %s" % tagged + return tagged[0]['task_id'] + + return None + def build(self, artifact_name, source): """ :param source : scmurl to spec repository : param artifact_name: name of artifact (which we couldn't get from spec due involved macros) :return koji build task id """ + + # This code supposes that artifact_name can be built within the component # Taken from /usr/bin/koji def _unique_path(prefix): - """Create a unique path fragment by appending a path component + """ + Create a unique path fragment by appending a path component to prefix. The path component will consist of a string of letter and numbers - that is unlikely to be a duplicate, but is not guaranteed to be unique.""" + that is unlikely to be a duplicate, but is not guaranteed to be unique. + """ # Use time() in the dirname to provide a little more information when # browsing the filesystem. # For some reason repr(time.time()) includes 4 or 5 # more digits of precision than str(time.time()) + # Unnamed Engineer: Guido v. R., I am disappoint return '%s/%r.%s' % (prefix, time.time(), - ''.join([random.choice(string.ascii_letters) for i in range(8)])) + ''.join([random.choice(string.ascii_letters) for i in range(8)])) if not self.__prep: raise RuntimeError("Buildroot is not prep-ed") + # Skip existing builds + task_id = self._get_task_by_artifact(artifact_name) + if task_id: + log.info("skipping build of %s. Build already exists (task_id=%s), via %s" % ( + source, task_id, self)) + return task_id + self._koji_whitelist_packages([artifact_name,]) if '://' not in source: #treat source as an srpm and upload it serverdir = _unique_path('cli-build') - callback =None + callback = None self.koji_session.uploadWrapper(source, serverdir, callback=callback) source = "%s/%s" % (serverdir, os.path.basename(source)) @@ -457,11 +554,23 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules def _koji_add_many_tag_inheritance(self, tag_name, parent_tags): tag = self._get_tag(tag_name) - - inheritanceData = [] + # highest priority num is at the end + inheritance_data = sorted(self.koji_session.getInheritanceData(tag['name']) or [], key=lambda k: k['priority']) + # Set initial priority to last record in inheritance data or 0 priority = 0 + if inheritance_data: + priority = inheritance_data[-1]['priority'] + 10 + def record_exists(parent_id, data): + for item in data: + if parent_id == item['parent_id']: + return True + return False + for parent in parent_tags: # We expect that they're sorted parent = self._get_tag(parent) + if record_exists(parent['id'], inheritance_data): + continue + parent_data = {} parent_data['parent_id'] = parent['id'] parent_data['priority'] = priority @@ -469,10 +578,11 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules parent_data['intransitive'] = False parent_data['noconfig'] = False parent_data['pkg_filter'] = '' - inheritanceData.append(parent_data) + inheritance_data.append(parent_data) priority += 10 - self.koji_session.setInheritanceData(tag['id'], inheritanceData) + if inheritance_data: + self.koji_session.setInheritanceData(tag['id'], inheritance_data) def _koji_add_groups_to_tag(self, dest_tag, groups=None): """ @@ -480,7 +590,6 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules :param groups: A dict {'group' : [package, ...]} """ log.debug("Adding groups=%s to tag=%s" % (groups.keys(), dest_tag)) - if groups and not isinstance(groups, dict): raise ValueError("Expected dict {'group' : [str(package1), ...]") @@ -493,34 +602,60 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules for group, packages in groups.iteritems(): group_id = existing_groups.get(group, None) if group_id is not None: - log.warning("Group %s already exists for tag %s" % (group, dest_tag)) + log.debug("Group %s already exists for tag %s. Skipping creation." % (group, dest_tag)) continue + self.koji_session.groupListAdd(dest_tag, group) log.debug("Adding %d packages into group=%s tag=%s" % (len(packages), group, dest_tag)) + + # This doesn't fail in case that it's already present in the group. This should be safe for pkg in packages: self.koji_session.groupPackageListAdd(dest_tag, group, pkg) - def _koji_create_tag(self, tag_name, arches=None, fail_if_exists=True, perm=None): - log.debug("Creating tag %s" % tag_name) - chktag = self.koji_session.getTag(tag_name) - if chktag and fail_if_exists: - raise SystemError("Tag %s already exist" % tag_name) + def _koji_create_tag(self, tag_name, arches=None, perm=None): + """ + :param tag_name: name of koji tag + :param arches: list of architectures for the tag + :param perm: permissions for the tag (used in lock-tag) - elif chktag: - return self._get_tag(self.module_tag) + This call is safe to call multiple times. + """ - else: - opts = {} - if arches: - if not isinstance(arches, list): - raise ValueError("Expected list or None on input got %s" % type(arches)) + log.debug("Ensuring existence of tag='%s'." % tag_name) + taginfo = self.koji_session.getTag(tag_name) + + if not taginfo: # Existing tag, need to check whether settings is correct + self.koji_session.createTag(tag_name, {}) + taginfo = self._get_tag(tag_name) + + opts = {} + if arches: + if not isinstance(arches, list): + raise ValueError("Expected list or None on input got %s" % type(arches)) + + current_arches = [] + if taginfo['arches']: # None if none + current_arches = taginfo['arches'].split() # string separated by empty spaces + + if set(arches) != set(current_arches): opts['arches'] = " ".join(arches) - self.koji_session.createTag(tag_name, **opts) - if perm: - self._lock_tag(tag_name, perm) - return self._get_tag(tag_name) + if perm: + if taginfo['locked']: + raise SystemError("Tag %s: master lock already set. Can't edit tag" % taginfo['name']) + + perm_ids = dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()]) + if perm not in perm_ids.keys(): + raise ValueError("Unknown permissions %s" % perm) + + perm_id = perm_ids[perm] + if taginfo['perm'] not in (perm_id, perm): # check either id or the string + opts['perm'] = perm_id + + # edit tag with opts + self.koji_session.editTag2(tag_name, **opts) + return self._get_tag(tag_name) # Return up2date taginfo def _get_component_owner(self, package): user = self.koji_session.getLoggedInUser()['name'] @@ -533,7 +668,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules for package in packages: package_id = pkglist.get(package, None) if not package_id is None: - log.warn("Package %s already exists in tag %s" % (package, self.module_tag['name'])) + log.debug("%s Package %s is already whitelisted." % (self, package)) continue to_add.append(package) @@ -545,20 +680,29 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules self.koji_session.packageListAdd(self.module_tag['name'], package, owner) def _koji_add_target(self, name, build_tag, dest_tag): + """ + :param name: target name + :param build-tag: build_tag name + :param dest_tag: dest tag name + + This call is safe to call multiple times. Raises SystemError() if the existing target doesn't match params. + The reason not to touch existing target, is that we don't want to accidentaly alter a target + which was already used to build some artifacts. + """ build_tag = self._get_tag(build_tag) dest_tag = self._get_tag(dest_tag) + target_info = self.koji_session.getBuildTarget(name) barches = build_tag.get("arches", None) - assert barches, "Build tag %s has no arches defined" % build_tag['name'] - self.koji_session.createBuildTarget(name, build_tag['name'], dest_tag['name']) - return self.koji_session.getBuildTarget(name) + assert barches, "Build tag %s has no arches defined." % build_tag['name'] - def _lock_tag(self, tag, perm="admin"): - taginfo = self._get_tag(tag) - if taginfo['locked']: - raise SystemError("Tag %s: master lock already set" % taginfo['name']) - perm_ids = dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()]) - if perm not in perm_ids.keys(): - raise ValueError("Unknown permissions %s" % perm) - perm_id = perm_ids[perm] - self.koji_session.editTag2(taginfo['id'], perm=perm_id) + if not target_info: + target_info = self.koji_session.createBuildTarget(name, build_tag['name'], dest_tag['name']) + + else: # verify whether build and destination tag matches + if build_tag['name'] != target_info['build_tag_name']: + raise SystemError("Target references unexpected build_tag_name. Got '%s', expected '%s'. Please contact administrator." % (target_info['build_tag_name'], build_tag['name'])) + if dest_tag['name'] != target_info['dest_tag_name']: + raise SystemError("Target references unexpected dest_tag_name. Got '%s', expected '%s'. Please contact administrator." % (target_info['dest_tag_name'], dest_tag['name'])) + + return self.koji_session.getBuildTarget(name) diff --git a/rida/scheduler/handlers/components.py b/rida/scheduler/handlers/components.py index d3a5f5f..9fb22f0 100644 --- a/rida/scheduler/handlers/components.py +++ b/rida/scheduler/handlers/components.py @@ -70,7 +70,7 @@ def _finalize(config, session, msg, state): module_name = parent.name tag = parent.koji_tag builder = rida.builder.KojiModuleBuilder(module_name, config, tag_name=tag) - builder.buildroot_resume() + builder.buildroot_connect() # tag && add to srpm-build group nvr = "{name}-{version}-{release}".format(**msg['msg']) install = bool(component_build.package == 'module-build-macros') diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 20942bd..8a1e1bf 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -101,9 +101,9 @@ def wait(config, session, msg): build.koji_tag = tag builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) - build.buildroot_task_id = builder.buildroot_prep() + build.buildroot_task_id = builder.buildroot_connect() log.debug("Adding dependencies %s into buildroot for module %s" % (dependencies, module_info)) - builder.buildroot_add_dependency(dependencies) + builder.buildroot_add_repos(dependencies) # inject dist-tag into buildroot srpm = builder.get_disttag_srpm(disttag=".%s" % get_rpm_release_from_tag(tag)) diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 8286e8a..100c055 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -76,7 +76,7 @@ def done(config, session, msg): return builder = rida.builder.KojiModuleBuilder(module_build.name, config, tag_name=tag) - builder.buildroot_resume() + builder.buildroot_connect() # Ok, for the subset of builds that did complete successfully, check to # see if they are in the buildroot. diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py index 12a9849..9d0a7d6 100644 --- a/tests/test_scheduler/test_repo_done.py +++ b/tests/test_scheduler/test_repo_done.py @@ -52,9 +52,9 @@ class TestRepoDone(unittest.TestCase): @mock.patch('rida.builder.KojiModuleBuilder.buildroot_ready') @mock.patch('rida.builder.KojiModuleBuilder.get_session_from_config') @mock.patch('rida.builder.KojiModuleBuilder.build') - @mock.patch('rida.builder.KojiModuleBuilder.buildroot_resume') + @mock.patch('rida.builder.KojiModuleBuilder.buildroot_connect') @mock.patch('rida.models.ModuleBuild.from_repo_done_event') - def test_a_single_match(self, from_repo_done_event, resume, build_fn, config, ready): + def test_a_single_match(self, from_repo_done_event, build_fn, config, ready): """ Test that when a repo msg hits us and we have a single match. """ config.return_value = mock.Mock(), "development"