From 21cd086d8337653338bc4c53f34e435e80ea032f Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Feb 03 2012 14:37:15 +0000 Subject: Improve Koji call performance with multiCall Fixes #311 --- diff --git a/lib/autoqa/koji_utils.py b/lib/autoqa/koji_utils.py index 01965ed..94047a8 100644 --- a/lib/autoqa/koji_utils.py +++ b/lib/autoqa/koji_utils.py @@ -40,51 +40,39 @@ class SimpleKojiClientSession(koji.ClientSession): return koji.ClientSession.__init__(self, server, opts) - def list_builds_since(self, timestamp): - '''Return a list of new builds since the given timestamp''' - return self.listBuilds(completeAfter=timestamp) - - - def list_tags(self, nvr): - '''Return a list of tags applied to the package with the given nvr''' - return [t['name'] for t in self.listTags(nvr)] - - def tag_history(self, nvr): '''Returns a list of every tag ever applied to the given nvr''' return [t['tag_name'] for t in self.tagHistory(build=nvr)] def latest_by_tag(self, tag, pkgname, max_evr=None): - '''Get the latest package for the given name in the given tag. If you + '''Get the latest Koji build for the given name in the given tag. If you set max_evr, it is *exclusive*. - max_evr = (epoch, version, release) + @type max_evr (epoch, version, release) + @return latest Koji build or None if there is no such build ''' - # allow epoch to be empty, transcode it to zero - if max_evr and not max_evr[0]: - max_evr = (0, max_evr[1], max_evr[2]) - if max_evr: - pkgs = self.listTagged(tag, package=pkgname) + builds = self.listTagged(tag, package=pkgname) # sort the packages from highest EVR to lowest - pkgs.sort(cmp=lambda x, y: compareEVR(x, y), reverse=True) - for pkg in pkgs[:]: - if rpmUtils.miscutils.compareEVR((pkg['epoch'] or '0', - pkg['version'], pkg['release']), max_evr) >= 0: - # remove all packages greater or equal than max_evr - pkgs.remove(pkg) - else: - # since packages are sorted, we can stop here - break + builds = sort_by_EVR(builds, ascending=False, max_evr=max_evr) else: - pkgs = self.listTagged(tag, package=pkgname, latest=True) - if pkgs: + builds = self.listTagged(tag, package=pkgname, latest=True) + + if builds: # the winner is the first package in list - return pkgs[0] + return builds[0] + + def build_failed(self, build): + '''Tell whether provided Koji build failed. + @param build either NVR (string) or Koji build object (dict) + @return True if build failed, False if it didn't (succeeded, or not yet + finished, or who knows what else) + ''' + if isinstance(build, str): + build = self.getBuild(build) - def build_failed(self, nvr): - return (self.getBuild(nvr)['state'] > 1) + return (build['state'] > 1) # Idea taken from the koji CLI @@ -99,21 +87,10 @@ class SimpleKojiClientSession(koji.ClientSession): print >> sys.stderr, "Warning: Koji API mismatch (Server %d, client %d)" % (ret,koji.API_VERSION) - def test_list_builds_since(self): - builds = [] - starttime = time.time() - timestamp = time.time() - while not builds and (starttime - timestamp < 10000): - timestamp -= 3000 - builds = self.list_builds_since(timestamp) - print "Most recent builds: %.0f min ago" % ((starttime-timestamp)/60.0) - return builds - - def list_previous_releases(self, name, tag, max_evr=None, unstable_tags=False): - '''Get most recent releases of package in specified tag (inheritence - enabled). Returns dictionary in the format {tag: NVR object}. Optional + '''Get most recent releases of package in the specified tag and its + parents. Returns dictionary in the format {tag: NVR object}. Optional max_evr says that all returned release versions must be lesser then it. If you enable unstable_tags, then also '-testing' tags are considered. @@ -220,30 +197,12 @@ class SimpleKojiClientSession(koji.ClientSession): rpm_files = [] print 'Fetching RPMs for: %s' % nvr for url in rpm_urls: - rpm_file = self.download_rpm(url, rpm_dir) + rpm_file = util.download(url, rpm_dir) rpm_files.append(rpm_file) return rpm_files - def download_rpm(self, url, rpm_dir): - '''Download a single RPM. - - @param url url of the RPM - @param rpm_dir location where to store the RPM - @return local filename of the grabbed RPM - @raise urlgrabber.grabber.URLGrabError if downloading failed - ''' - - # create destination dir if needed - util.makedirs(rpm_dir) - - # download - rpm_file = util.download(url, dest=rpm_dir) - - return rpm_file - - def compareEVR(nvr1, nvr2): '''Compare EVR from two NVR objects. @param nvr1 Koji build object or None @@ -271,3 +230,35 @@ def getENVR(build): return '%s:%s' % (build['epoch'], build['nvr']) else: return build['nvr'] + + +def sort_by_EVR(builds, ascending=True, max_evr=None): + '''Sort list of Koji builds by EVR (of course this makes sense only if + all of them have the same name). Filter them if required. + + @param ascending sort from lowest to highest if True, from highest to + lowest if False + @param max_evr filter out all packages with greater or equal EVR than + provided one + @type max_evr (epoch, version, release) + @return list of sorted and filtered Koji builds + ''' + # sort packages + builds.sort(cmp=lambda x, y: compareEVR(x, y), reverse=not ascending) + + if max_evr: + # allow epoch to be empty, transcode it to zero + if not max_evr[0]: + max_evr = (0, max_evr[1], max_evr[2]) + + for build in builds[:]: + if (rpmUtils.miscutils.compareEVR( + (build['epoch'] or '0', build['version'], build['release']), + max_evr) >= 0): + # remove all packages greater or equal than max_evr + builds.remove(build) + else: + # since packages are sorted, we can stop here + break + + return builds diff --git a/tests/rpmguard/rpmguard.py b/tests/rpmguard/rpmguard.py index e45d283..b6ff8da 100644 --- a/tests/rpmguard/rpmguard.py +++ b/tests/rpmguard/rpmguard.py @@ -131,8 +131,8 @@ class rpmguard(AutoQATest): for old_rpm, new_rpm in rpm_to_match: # fetch old and new rpms - old_file = self.koji.download_rpm(old_rpm['url'], self.tmpdir) - new_file = self.koji.download_rpm(new_rpm['url'], self.tmpdir) + old_file = util.download(old_rpm['url'], self.tmpdir) + new_file = util.download(new_rpm['url'], self.tmpdir) # run rpmguard cmd = '%s %s %s' % (self.rpmguard, old_file, new_file) diff --git a/tests/upgradepath/upgradepath.py b/tests/upgradepath/upgradepath.py index 3cf983e..f739ad5 100755 --- a/tests/upgradepath/upgradepath.py +++ b/tests/upgradepath/upgradepath.py @@ -57,7 +57,7 @@ class upgradepath(AutoQATest): self.koji.check_connection() - def compare(self, proposed_build, tags, op, test_detail, test_pending=False): + def compare(self, proposed_build, tags, op, tag_builds, test_detail, test_pending=False): '''Compare proposed_build with given operator to latest build in all tags and update test_detail.result. @param proposed_build Koji Build object @@ -67,6 +67,8 @@ class upgradepath(AutoQATest): Example: ['f15', 'f16-updates'] or [('f15',), ('f16', 'f16-updates')] @param op comparison operator from @module operator + @param tag_builds dictionary that maps every available tag to the latest + Koji build object (or None) available for this tag @param test_detail TestDetail object related to the proposed build @param test_pending look also into Bodhi for pending requests into stable and consider it in the algorithm @@ -97,7 +99,7 @@ class upgradepath(AutoQATest): # find the latest build in repositories latest_main_build = None for tag in tag_tuple: - build = self.koji.latest_by_tag(tag, proposed_build['name']) + build = tag_builds[tag] if koji_utils.compareEVR(build, latest_main_build) > 0: latest_main_build = build @@ -109,7 +111,7 @@ class upgradepath(AutoQATest): # find the completely latest build latest_build = latest_main_build if (test_pending and - koji_utils.compareEVR(latest_pend_build, latest_build) > 0): + koji_utils.compareEVR(latest_pend_build, latest_build) > 0): latest_build = latest_pend_build # compare and find out the result @@ -375,10 +377,17 @@ class upgradepath(AutoQATest): for build, update in build2update.iteritems(): self.build2detail[build] = update2detail[update] + # Get build info from Koji for every ENVR + # Use multicall to speed it up + self.log('Getting build information from Koji...') + self.koji.multicall = True + for envr in envrs: + self.koji.getBuild(envr) + koji_builds = self.koji.multiCall() # Let the testing begin! self.log('Running main upgradepath tests...') - for envr in envrs: + for (envr, koji_build) in zip(envrs, koji_builds): detail = self.build2detail[envr] if detail.broken(): # this Bodhi update was already broken by some envr, skip it @@ -387,14 +396,15 @@ class upgradepath(AutoQATest): self.log('%s\n%s into %s\n%s' % (60*'=', envr, kojitag, 60*'='), details=detail) - # get all info about this current build - proposed_build = self.koji.getBuild(envr) - if proposed_build is None: - msg = 'This build does not exist in Koji: %s' % envr + # Koji returns list if the call succeeded and dict (containing error + # information) if the call failed + if isinstance(koji_build, dict): + msg = "Can't fetch info about %s from Koji: %s" % (envr, koji_build) self.log(msg, highlight=True, stderr=True, details=detail) self.problematic[envr] = msg detail.update_result('ABORTED') continue + proposed_build = koji_build[0] # if we want to push an older build, warn about that and perform further # checking for the most recent one @@ -408,26 +418,47 @@ class upgradepath(AutoQATest): self.log(msg, highlight=True, stderr=True, details=detail) proposed_build = latest_build - # start the main testing phase + # query Koji for build information for all packages using multicall + all_tags = [repo['tag'] for repo in repos] + self.koji.multicall = True + for tag in all_tags: + self.koji.listTagged(tag, package=proposed_build['name'], + latest=True) + values = self.koji.multiCall() + tag_builds = {} + + for tag, value in zip(all_tags, values): + # value is dict if call failed + if isinstance(value, dict): + self.log('Error querying Koji for %s: %s' % (tag, value), + highlight=True, stderr=True) + tag_builds[tag] = None + else: + # value is now [[koji_build]] or [[]] + tag_builds[tag] = value[0][0] if value[0] else None + + + ### start the main testing phase ### nvr_result = TestDetail(test=self,id=None) + if push_to_main: # compare with lower tags, so version has to be greater or equal low_tags = [repo['tag'] for repo in low_repos] result = self.compare(proposed_build, low_tags, operator.ge, - detail) + tag_builds, detail) nvr_result.update_result(result) # compare with higher tags, so version has to be lower or equal hi_tags = [repo['tag'] for repo in hi_repos] result = self.compare(proposed_build, hi_tags, operator.le, - detail) + tag_builds, detail) nvr_result.update_result(result) else: # pushing to update repository # compare with lower tags, so version has to be greater or equal low_tags = [repo['tag'] for repo in low_repos] result = self.compare(proposed_build, low_tags, operator.ge, - detail) + tag_builds, detail) nvr_result.update_result(result) # compare with higher tags, so version has to be lower or equal @@ -460,7 +491,7 @@ class upgradepath(AutoQATest): # now do the comparison finally # this time we also want to check for pending Bodhi requests result = self.compare(proposed_build, hi_tags, operator.le, - detail, test_pending=True) + tag_builds, detail, test_pending=True) nvr_result.update_result(result) self.log('RESULT: %s' % nvr_result.result, details=detail) diff --git a/watchers/koji-bodhi/watcher.py b/watchers/koji-bodhi/watcher.py index 802c839..6a8fd95 100755 --- a/watchers/koji-bodhi/watcher.py +++ b/watchers/koji-bodhi/watcher.py @@ -180,9 +180,20 @@ class KojiWatcher(object): # sadly the tagHistory does not all the required information # we need to run the tests, so let's fetch the 'good' info logging.info(" Getting real info about builds") - for i in range(len(updates)): - b = updates[i] - updates[i] = self.session.getBuild("%s-%s-%s" % (b['name'], b['version'], b['release'])) + self.session.multicall = True + for update in updates: + self.session.getBuild("%s-%s-%s" % (update['name'], update['version'], + update['release'])) + koji_builds = self.session.multiCall() + + for i, build in enumerate(koji_builds): + # if call failed, we receive a dict with details + if isinstance(build, dict): + logging.error("Can't get Koji info about %s: %s" % + (koji_utils.getENVR(updates[i]), build)) + continue + + updates[i] = build[0] # TODO: maybe this is bug in KOJI - check it out # because getBuild returns 'id' instead of 'build_id' updates[i]['build_id'] = updates[i]['id'] @@ -210,7 +221,7 @@ class KojiWatcher(object): logging.info(" Fetching builds since: %s", prevtime) - builds = self.session.list_builds_since(prevtime) + builds = self.session.listBuilds(completeAfter=prevtime) try: builds += self.prevtimes['untagged_builds'] # untagged builds from previous run logging.info(" Adding previously stored untagged builds") @@ -228,15 +239,25 @@ class KojiWatcher(object): pass logging.info(" Assorting according to tags") - for nvr in builds: - tags = set(self.session.tag_history(nvr)) + self.session.multicall = True + for build in builds: + self.session.tagHistory(build) + for build in builds: + self.session.getBuild(build) + response = self.session.multiCall(strict=True) + + history = response[:len(builds)] + build_infos = response[len(builds):] + + for i, build in enumerate(builds): + tags = set([t['tag_name'] for t in history[i][0]]) if tags: # Only check packages with tags from the taglist for t in taglist.intersection(tags): - tagged_builds[t].append(nvr) - elif not self.session.build_failed(nvr): + tagged_builds[t].append(build) + elif not self.session.build_failed(build_infos[i][0]): # No tag yet but the build is still OK - check again later - untagged_builds.append(nvr) + untagged_builds.append(build) logging.info(" Assorting done") self.prevtimes['untagged_builds'] = untagged_builds @@ -306,7 +327,7 @@ class KojiWatcher(object): i.e. calling the harness in the required way. This should be the only method to reimplement in all - the different post-koji wathers (i.e. the batch watcher etc.) + the different post-koji watchers (i.e. the batch watcher etc.) Or we can implement several 'schedule_jobs' methods in this particular instance, and run them in the run() method. @@ -317,10 +338,16 @@ class KojiWatcher(object): # schedule koji events for tag in sorted(new_builds.keys()): bodhi_skip = [] - logging.info("Scheduling tests for %s" % tag) + + self.session.multicall = True for b in new_builds[tag]: + self.session.listRPMs(b['build_id']) + rpms = self.session.multiCall(strict=True) + + logging.info("Scheduling tests for %s" % tag) + for (b, b_rpms) in zip(new_builds[tag], rpms): # Get a list of all package arches in this build - arches = [r['arch'] for r in self.session.listRPMs(b['build_id'])] + arches = [r['arch'] for r in b_rpms[0]] arches = map(get_basearch, arches) harnesscall = ['autoqa'] p_tag = tag.replace('-pending', '') @@ -409,11 +436,16 @@ class KojiWatcher(object): p_tag = tag.replace('-pending', '') repoarches = set(repoinfo.getrepo_by_tag(p_tag).get("arches")) + self.session.multicall = True for b in new_builds[tag]: + self.session.listRPMs(b['build_id']) + rpms = self.session.multiCall(strict=True) + + for (b, b_rpms) in zip(new_builds[tag], rpms): # do not query koji for arches, if we already have rpms from all # possible arches from the repoarches if harness_arches != repoarches: - arches = [r['arch'] for r in self.session.listRPMs(b['build_id'])] + arches = [r['arch'] for r in b_rpms[0]] arches = map(get_basearch, arches) testarches = set(repoarches).intersection(arches) harness_arches.update(testarches) @@ -515,7 +547,7 @@ tags for new builds and kick off tests when new builds/packages are found.') # FIXME - lock.acquire() will fail if cachedir doesn't exist. While # cachedir is created in KojiWatcher.__init__(), checking for the # presence of cachedir below should offer additional protection. - if os.path.exists(KojiWatcher.cachedir) and lock.acquire() is not True: + if os.path.exists(KojiWatcher.cachedir) and not lock.acquire(): logging.warn("Another instance of this watcher is already running. Exiting...") sys.exit(0)