From fcf84dd6221bf1773b861baf1ce421dc983681be Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 25 2020 14:09:55 +0000 Subject: [PATCH 1/4] move checkTasks near its usage Related: https://pagure.io/koji/issue/2119 Fixes: https://pagure.io/koji/issue/2581 --- diff --git a/util/kojira b/util/kojira index bca7d8d..0785421 100755 --- a/util/kojira +++ b/util/kojira @@ -712,11 +712,6 @@ class RepoManager(object): self.setTagScore(entry) def updateRepos(self): - self.checkTasks() - self.logger.debug("Current tasks: %r" % self.tasks) - if self.other_tasks: - self.logger.debug("Found %i untracked newRepo tasks", - len(self.other_tasks)) self.logger.debug("Updating repos") self.readCurrentRepos() @@ -875,6 +870,12 @@ class RepoManager(object): def regenRepos(self): """Trigger newRepo tasks for needed tags""" + self.checkTasks() + self.logger.debug("Current tasks: %r" % self.tasks) + if self.other_tasks: + self.logger.debug("Found %i untracked newRepo tasks", + len(self.other_tasks)) + # first note currently running tasks running_tasks = 0 running_tasks_maven = 0 From 2b032743a2701764b3de181d6dac2a7f55f1c374 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 25 2020 14:10:11 +0000 Subject: [PATCH 2/4] remove recent tasks behaviour --- diff --git a/util/kojira b/util/kojira index 0785421..51eceda 100755 --- a/util/kojira +++ b/util/kojira @@ -277,7 +277,6 @@ class RepoManager(object): self.repos = {} self.external_repo_ts = {} self.tasks = {} - self.recent_tasks = {} self.other_tasks = {} self.needed_tags = {} self.tag_use_stats = {} @@ -745,9 +744,6 @@ class RepoManager(object): # prune recent tasks now = time.time() - for task_id in list(self.recent_tasks): - if now - self.recent_tasks[task_id] > self.options.recent_tasks_lifetime: - del self.recent_tasks[task_id] # check on current tasks task_ids = list(self.tasks) @@ -759,12 +755,11 @@ class RepoManager(object): tag_id = self.tasks[task_id]['tag_id'] if tstate == 'CLOSED': self.logger.info("Finished: newRepo task %s for tag %s", task_id, tag_id) - self.recent_tasks[task_id] = time.time() del self.tasks[task_id] + del self.needed_tags[tag_id] elif tstate in ('CANCELED', 'FAILED'): self.logger.info( "Problem: newRepo task %s for tag %s is %s", task_id, tag_id, tstate) - self.recent_tasks[task_id] = time.time() del self.tasks[task_id] else: self.tasks[task_id]['taskinfo'] = tinfo @@ -910,9 +905,6 @@ class RepoManager(object): if task_id in self.tasks: # we already have a task continue - elif task_id in self.recent_tasks: - # avoiding a race, see https://pagure.io/koji/issue/942 - continue else: # should not happen logger.warning('Needed tag refers to unknown task. ' @@ -1135,7 +1127,6 @@ def get_options(): # XXX should really be called expired_repo_lifetime 'dist_repo_lifetime': 7 * 24 * 3600, 'check_external_repos': False, - 'recent_tasks_lifetime': 600, 'sleeptime': 15, 'cert': None, 'serverca': None, @@ -1145,13 +1136,13 @@ def get_options(): int_opts = ('deleted_repo_lifetime', 'max_repo_tasks', 'repo_tasks_limit', 'retry_interval', 'max_retries', 'offline_retry_interval', 'max_delete_processes', 'max_repo_tasks_maven', 'dist_repo_lifetime', - 'sleeptime', 'recent_tasks_lifetime') + 'sleeptime') str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'cert', 'serverca', 'debuginfo_tags', 'queue_file', 'source_tags', 'separate_source_tags', 'ignore_tags') bool_opts = ('verbose', 'debug', 'ignore_stray_repos', 'offline_retry', 'no_ssl_verify', 'check_external_repos') - legacy_opts = ('with_src', 'delete_batch_size') + legacy_opts = ('with_src', 'delete_batch_size', 'recent_tasks_lifetime') for name in config.options(section): if name in int_opts: defaults[name] = config.getint(section, name) From cfc2632fa914b78fee7bec36aaa2b4824da8ede4 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 25 2020 14:10:11 +0000 Subject: [PATCH 3/4] allow other thread to remove tag --- diff --git a/util/kojira b/util/kojira index 51eceda..ae360fe 100755 --- a/util/kojira +++ b/util/kojira @@ -756,7 +756,11 @@ class RepoManager(object): if tstate == 'CLOSED': self.logger.info("Finished: newRepo task %s for tag %s", task_id, tag_id) del self.tasks[task_id] - del self.needed_tags[tag_id] + try: + del self.needed_tags[tag_id] + except KeyError: + # it could be already removed by checkNeeded + pass elif tstate in ('CANCELED', 'FAILED'): self.logger.info( "Problem: newRepo task %s for tag %s is %s", task_id, tag_id, tstate) @@ -811,7 +815,11 @@ class RepoManager(object): # no longer needed self.logger.info("Tag %(name)s has a current or in " "progress repo", entry['taginfo']) - del self.needed_tags[tag_id] + try: + del self.needed_tags[tag_id] + except KeyError: + # it could be already deleted by checkTasks + pass # if not covered, we already know continue if covered: From 5dcf1889aff2db3234cf7c833435a6226949883f Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 25 2020 14:10:11 +0000 Subject: [PATCH 4/4] remove unused variable --- diff --git a/util/kojira b/util/kojira index ae360fe..5a207e1 100755 --- a/util/kojira +++ b/util/kojira @@ -742,9 +742,6 @@ class RepoManager(object): - check for other newRepo tasks (not generated by us) """ - # prune recent tasks - now = time.time() - # check on current tasks task_ids = list(self.tasks) self.session.multicall = True