From 8a13ffa219418031ffb6b157911fa0cab9028ae7 Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Jun 16 2022 07:29:26 +0000 Subject: Download output for all type of task in download-task Fixes: https://pagure.io/koji/issue/3182 --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index d1a4612..295d649 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -7325,7 +7325,8 @@ def anon_handle_download_task(options, session, args): usage = "usage: %prog download-task " parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--arch", dest="arches", metavar="ARCH", action="append", default=[], - help="Only download packages for this arch (may be used multiple times)") + help="Only download packages for this arch (may be used multiple times), " + "only for build and buildArch task methods") parser.add_option("--logs", dest="logs", action="store_true", default=False, help="Also download build logs") parser.add_option("--topurl", metavar="URL", default=options.topurl, @@ -7337,6 +7338,10 @@ def anon_handle_download_task(options, session, args): help="Do not wait for running tasks to finish") parser.add_option("-q", "--quiet", action="store_true", help="Suppress output", default=options.quiet) + parser.add_option("--all", action="store_true", + help="Download all files, all methods instead of build and buildArch") + parser.add_option("--dirpertask", action="store_true", help="Download files to dir per task") + parser.add_option("--parentonly", action="store_true", help="Download parent's files only") (suboptions, args) = parser.parse_args(args) if len(args) == 0: @@ -7364,40 +7369,78 @@ def anon_handle_download_task(options, session, args): watch_tasks(session, [base_task_id], quiet=suboptions.quiet, poll_interval=options.poll_interval, topurl=options.topurl) - def check_downloadable(task): - return task["method"] == "buildArch" - - downloadable_tasks = [] + list_tasks = [base_task] + if not suboptions.parentonly: + list_tasks.extend(session.getTaskChildren(base_task_id)) - if check_downloadable(base_task): - downloadable_tasks.append(base_task) - else: - subtasks = session.getTaskChildren(base_task_id) - downloadable_tasks.extend(list(filter(check_downloadable, subtasks))) + # support file types + expected_types = ['rpm', 'log'] + for type in session.getArchiveTypes(): + expected_types.extend(type['extensions'].split(' ')) # get files for download downloads = [] - - for task in downloadable_tasks: - files = list_task_output_all_volumes(session, task["id"]) - for filename in files: - if filename.endswith(".rpm"): - for volume in files[filename]: - filearch = filename.split(".")[-2] - if len(suboptions.arches) == 0 or filearch in suboptions.arches: - downloads.append((task, filename, volume, filename)) - elif filename.endswith(".log") and suboptions.logs: - for volume in files[filename]: - # rename logs, they would conflict - new_filename = "%s.%s.log" % (filename.rstrip(".log"), task["arch"]) - downloads.append((task, filename, volume, new_filename)) + build_methods_list = ['buildArch', 'build'] + for task in list_tasks: + taskarch = task['arch'] + task_id = str(task['id']) + if len(suboptions.arches) == 0 or taskarch in suboptions.arches: + files = list_task_output_all_volumes(session, task["id"]) + filetype = None + for filename in files: + if filename.endswith('src.rpm'): + filetype = 'src.rpm' + else: + for ft in expected_types: + if filename.endswith('.%s' % ft): + filetype = ft + break + if not filetype: + warn('Unsupported file type for download-task: %s' % filename) + else: + if suboptions.all and task['method'] not in build_methods_list: + if filetype != 'log': + for volume in files[filename]: + if suboptions.dirpertask: + new_filename = '%s/%s' % (task_id, filename) + else: + if taskarch not in filename and filetype != 'src.rpm': + part_filename = filename[:-len('.%s' % filetype)] + new_filename = "%s.%s.%s" % (part_filename, + taskarch, filetype) + else: + new_filename = filename + downloads.append((task, filename, volume, new_filename, task_id)) + elif task['method'] in build_methods_list: + if filetype in ['rpm', 'src.rpm']: + filearch = filename.split(".")[-2] + for volume in files[filename]: + if len(suboptions.arches) == 0 or filearch in suboptions.arches: + if suboptions.dirpertask: + new_filename = '%s/%s' % (task_id, filename) + else: + new_filename = filename + downloads.append((task, filename, volume, new_filename, + task_id)) + + if filetype == 'log' and suboptions.logs: + for volume in files[filename]: + if suboptions.dirpertask: + new_filename = '%s/%s' % (task_id, filename) + else: + if taskarch not in filename: + part_filename = filename[:-len('.log')] + new_filename = "%s.%s.log" % (part_filename, taskarch) + else: + new_filename = filename + downloads.append((task, filename, volume, new_filename, task_id)) if len(downloads) == 0: print("No files for download found.") return required_tasks = {} - for (task, nop, nop, nop) in downloads: + for (task, nop, nop, nop, nop) in downloads: if task["id"] not in required_tasks: required_tasks[task["id"]] = task @@ -7408,14 +7451,26 @@ def anon_handle_download_task(options, session, args): else: error("Child task %d has not finished yet." % task_id) + downloads_new_names = [(new_filename, vol) for (_, _, vol, new_filename, _) in downloads] + if not suboptions.dirpertask: + not_uniques = list({x for x in downloads_new_names if downloads_new_names.count(x) > 1}) + if not_uniques: + error("Download files names conflict, use --dirpertask") + # perform the download number = 0 pathinfo = koji.PathInfo(topdir=suboptions.topurl) - for (task, filename, volume, new_filename) in downloads: + for (task, filename, volume, new_filename, task_id) in downloads: + if suboptions.dirpertask: + koji.ensuredir(task_id) number += 1 if volume not in (None, 'DEFAULT'): - koji.ensuredir(volume) - new_filename = os.path.join(volume, new_filename) + if suboptions.dirpertask: + koji.ensuredir('%s/%s' % (task_id, volume)) + new_filename = os.path.join(task_id, volume, filename) + else: + koji.ensuredir(volume) + new_filename = os.path.join(volume, new_filename) if '..' in filename: error('Invalid file name: %s' % filename) url = '%s/%s/%s' % (pathinfo.work(volume), pathinfo.taskrelpath(task["id"]), filename) diff --git a/tests/test_cli/test_download_build.py b/tests/test_cli/test_download_build.py index 3512b40..74ad2ff 100644 --- a/tests/test_cli/test_download_build.py +++ b/tests/test_cli/test_download_build.py @@ -10,6 +10,7 @@ from . import utils class TestDownloadBuild(utils.CliTestCase): def setUp(self): + self.maxDiff = None self.options = mock.MagicMock() self.options.debug = False self.session = mock.MagicMock() @@ -186,6 +187,48 @@ class TestDownloadBuild(utils.CliTestCase): exit_code=1 ) + def test_download_build_latest_from_error_find_build(self): + build_id = 'package-name' + self.session.listTagged.side_effect = koji.GenericError + self.assert_system_exit( + anon_handle_download_build, + self.options, + self.session, + [build_id, '--latestfrom', self.tag], + stderr='Error finding latest build: {}\n', + activate_session=None, + exit_code=1 + ) + + def test_download_build_topurl_none(self): + build_id = '1' + self.assert_system_exit( + anon_handle_download_build, + self.options, + self.session, + [build_id, '--topurl', None], + stderr='You must specify --topurl to download files\n', + activate_session=None, + exit_code=1 + ) + + @mock.patch('koji.buildLabel') + def test_download_build_type_not_scratch(self, build_label): + build_id = '1' + type = 'rpm' + self.session.listArchives.return_value = [] + nvr = self.build_templ['nvr'] + build_label.return_value = nvr + self.assert_system_exit( + anon_handle_download_build, + self.options, + self.session, + [build_id, '--type', type], + stderr='No %s archives available for %s\n' % (type, nvr), + activate_session=None, + exit_code=1 + ) + def test_handle_add_volume_help(self): self.assert_help( anon_handle_download_build, diff --git a/tests/test_cli/test_download_task.py b/tests/test_cli/test_download_task.py index 64d0a23..6d0ab3b 100644 --- a/tests/test_cli/test_download_task.py +++ b/tests/test_cli/test_download_task.py @@ -14,8 +14,6 @@ progname = os.path.basename(sys.argv[0]) or 'koji' class TestDownloadTask(utils.CliTestCase): - # Show long diffs in error output... - maxDiff = None def gen_calls(self, task_output, pattern, blacklist=[], arch=None): @@ -40,6 +38,8 @@ class TestDownloadTask(utils.CliTestCase): return calls def setUp(self): + # Show long diffs in error output... + self.maxDiff = None # Mock out the options parsed in main self.options = mock.MagicMock() self.options.quiet = None @@ -53,17 +53,22 @@ class TestDownloadTask(utils.CliTestCase): self.ensure_connection = mock.patch('koji_cli.commands.ensure_connection').start() self.stdout = mock.patch('sys.stdout', new_callable=six.StringIO).start() self.stderr = mock.patch('sys.stderr', new_callable=six.StringIO).start() + self.parent_task_id = 123333 + self.parent_task_info = {'id': self.parent_task_id, 'method': 'buildArch', + 'arch': 'taskarch', 'state': 2, 'parent': None} + self.error_format = """Usage: %s download-task +(Specify the --help global option for a list of other help options) + +%s: error: {message} +""" % (self.progname, self.progname) def tearDown(self): mock.patch.stopall() def test_handle_download_task_single(self): - task_id = 123333 - args = [str(task_id)] - self.session.getTaskInfo.return_value = {'id': task_id, - 'method': 'buildArch', - 'arch': 'taskarch', - 'state': 2} + args = [str(self.parent_task_id)] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [] self.list_task_output_all_volumes.return_value = { 'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'], @@ -84,41 +89,35 @@ class TestDownloadTask(utils.CliTestCase): self.assertMultiLineEqual(actual, expected) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) - self.session.getTaskChildren.assert_not_called() - self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.list_task_output_all_volumes.assert_called_once_with(self.session, + self.parent_task_id) self.assertListEqual(self.download_file.mock_calls, calls) self.assertIsNone(rv) def test_handle_download_task_not_found(self): - task_id = 123333 - args = [str(task_id)] + args = [str(self.parent_task_id)] self.session.getTaskInfo.return_value = None # Run it and check immediate output # args: task_id # expected: error - with self.assertRaises(SystemExit) as ex: - anon_handle_download_task(self.options, self.session, args) - self.assertExitCode(ex, 1) - actual = self.stdout.getvalue() - expected = '' - self.assertMultiLineEqual(actual, expected) - actual = self.stderr.getvalue() - expected = 'No such task: %s\n' % task_id - self.assertMultiLineEqual(actual, expected) + self.assert_system_exit( + anon_handle_download_task, + self.options, self.session, args, + stderr='No such task: %s\n' % self.parent_task_id, + stdout='', + activate_session=None, + exit_code=1) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) self.session.getTaskChildren.assert_not_called() def test_handle_download_task_parent(self): - task_id = 123333 - args = [str(task_id), '--arch=noarch,x86_64'] - self.session.getTaskInfo.return_value = {'id': task_id, - 'method': 'build', - 'arch': 'taskarch', - 'state': 2} + args = [str(self.parent_task_id), '--arch=noarch,x86_64'] + self.session.getTaskInfo.return_value = self.parent_task_info self.session.getTaskChildren.return_value = [{'id': 22222, 'method': 'buildArch', 'arch': 'noarch', @@ -140,7 +139,8 @@ class TestDownloadTask(utils.CliTestCase): {'somerpm.src.rpm': ['DEFAULT', 'vol1']}, {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']}, {'somerpm.noarch.rpm': ['vol3'], - 'somelog.log': ['DEFAULT', 'vol1']}] + 'somelog.log': ['DEFAULT', 'vol1']}, + ] # Run it and check immediate output # args: task_id --arch=noarch,x86_64 # expected: success @@ -151,36 +151,31 @@ class TestDownloadTask(utils.CliTestCase): self.assertMultiLineEqual(actual, expected) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) - self.session.getTaskChildren.assert_called_once_with(task_id) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ call(self.session, 22222), call(self.session, 33333), - call(self.session, 44444)]) + call(self.session, 55555)]) self.assertListEqual(self.download_file.mock_calls, [ call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm', - 'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=1), + 'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=2, num=1), call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm', - 'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=2), - call('https://topurl/vol/vol3/work/tasks/4444/44444/somerpm.noarch.rpm', - 'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=3, num=3)]) + 'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=2, num=2)]) self.assertIsNone(rv) def test_handle_download_task_log(self): - task_id = 123333 - args = [str(task_id), '--log'] - self.session.getTaskInfo.return_value = {'id': task_id, - 'method': 'buildArch', - 'arch': 'taskarch', - 'state': 2} - self.list_task_output_all_volumes.return_value = { + args = [str(self.parent_task_id), '--log'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [{'id': 22222, + 'method': 'buildArch', + 'arch': 'noarch', + 'state': 2}] + self.list_task_output_all_volumes.side_effect = [{}, { 'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'], 'somerpm.noarch.rpm': ['vol3'], - 'somelog.log': ['DEFAULT', 'vol1']} - - calls = self.gen_calls(self.list_task_output_all_volumes.return_value, - 'https://topurl/%swork/tasks/3333/123333/%s', arch='taskarch') + 'somelog.log': ['DEFAULT', 'vol1']}] # Run it and check immediate output # args: task_id --log @@ -192,26 +187,31 @@ class TestDownloadTask(utils.CliTestCase): self.assertMultiLineEqual(actual, expected) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) - self.session.getTaskChildren.assert_not_called() - self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id) - self.assertListEqual(self.download_file.mock_calls, calls) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.list_task_output_all_volumes.assert_has_calls([ + mock.call(self.session, self.parent_task_id), mock.call(self.session, 22222)]) + self.assertListEqual(self.download_file.mock_calls, [ + call('https://topurl/work/tasks/2222/22222/somerpm.src.rpm', + 'somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=1), + call('https://topurl/vol/vol1/work/tasks/2222/22222/somerpm.src.rpm', + 'vol1/somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=2), + call('https://topurl/work/tasks/2222/22222/somerpm.x86_64.rpm', + 'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=3), + call('https://topurl/vol/vol2/work/tasks/2222/22222/somerpm.x86_64.rpm', + 'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=4), + call('https://topurl/vol/vol3/work/tasks/2222/22222/somerpm.noarch.rpm', + 'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=7, num=5), + call('https://topurl/work/tasks/2222/22222/somelog.log', + 'somelog.noarch.log', quiet=None, noprogress=None, size=7, num=6), + call('https://topurl/vol/vol1/work/tasks/2222/22222/somelog.log', + 'vol1/somelog.noarch.log', quiet=None, noprogress=None, size=7, num=7) + ]) self.assertIsNone(rv) def test_handle_download_no_download(self): - task_id = 123333 - args = [str(task_id), '--arch=s390,ppc'] - self.session.getTaskInfo.return_value = {'id': task_id, - 'method': 'buildArch', - 'arch': 'taskarch', - 'state': 2} - self.list_task_output_all_volumes.return_value = { - 'somerpm.src.rpm': ['DEFAULT', 'vol1'], - 'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'], - 'somerpm.noarch.rpm': ['vol3'], - 'somelog.log': ['DEFAULT', 'vol1'], - 'somezip.zip': ['DEFAULT'] - } + args = [str(self.parent_task_id), '--arch=s390,ppc'] + self.session.getTaskInfo.return_value = self.parent_task_info # Run it and check immediate output # args: task_id --arch=s390,ppc @@ -225,15 +225,14 @@ class TestDownloadTask(utils.CliTestCase): self.assertMultiLineEqual(actual, expected) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) - self.session.getTaskChildren.assert_not_called() - self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.list_task_output_all_volumes.assert_not_called() self.download_file.assert_not_called() def test_handle_download_parent_not_finished(self): - task_id = 123333 - args = [str(task_id)] - self.session.getTaskInfo.return_value = {'id': task_id, + args = [str(self.parent_task_id)] + self.session.getTaskInfo.return_value = {'id': self.parent_task_id, 'method': 'buildArch', 'arch': 'taskarch', 'state': 3} @@ -247,78 +246,69 @@ class TestDownloadTask(utils.CliTestCase): # Run it and check immediate output # args: task_id # expected: failure - with self.assertRaises(SystemExit) as ex: - anon_handle_download_task(self.options, self.session, args) - self.assertExitCode(ex, 1) - actual = self.stdout.getvalue() - expected = '' - self.assertMultiLineEqual(actual, expected) - actual = self.stderr.getvalue() - expected = 'Task 123333 has not finished yet.\n' - self.assertMultiLineEqual(actual, expected) + self.assert_system_exit( + anon_handle_download_task, + self.options, self.session, args, + stderr="Task 123333 has not finished yet.\n", + stdout='', + activate_session=None, + exit_code=1) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) - self.session.getTaskChildren.assert_not_called() - self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.list_task_output_all_volumes.assert_called_once_with( + self.session, self.parent_task_id) self.download_file.assert_not_called() def test_handle_download_child_not_finished(self): - task_id = 123333 - args = [str(task_id)] - self.session.getTaskInfo.return_value = {'id': task_id, - 'method': 'build', - 'arch': 'taskarch', - 'state': 2} + args = [str(self.parent_task_id)] + self.session.getTaskInfo.return_value = self.parent_task_info self.session.getTaskChildren.return_value = [{'id': 22222, 'method': 'buildArch', 'arch': 'noarch', 'state': 3}] - self.list_task_output_all_volumes.return_value = {'somerpm.src.rpm': ['DEFAULT', 'vol1']} + self.list_task_output_all_volumes.side_effect = [ + {'somerpm.src.rpm': ['DEFAULT', 'vol1']}, + {'somenextrpm.src.rpm': ['DEFAULT', 'vol1']}] # Run it and check immediate output # args: task_id # expected: failure - with self.assertRaises(SystemExit) as ex: - anon_handle_download_task(self.options, self.session, args) - self.assertExitCode(ex, 1) - actual = self.stdout.getvalue() - expected = '' - self.assertMultiLineEqual(actual, expected) - actual = self.stderr.getvalue() - expected = 'Child task 22222 has not finished yet.\n' - self.assertMultiLineEqual(actual, expected) + self.assert_system_exit( + anon_handle_download_task, + self.options, self.session, args, + stderr="Child task 22222 has not finished yet.\n", + stdout='', + activate_session=None, + exit_code=1) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) - self.session.getTaskChildren.assert_called_once_with(task_id) - self.list_task_output_all_volumes.assert_called_once_with(self.session, 22222) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.list_task_output_all_volumes.assert_has_calls( + [mock.call(self.session, self.parent_task_id), mock.call(self.session, 22222)]) self.download_file.assert_not_called() def test_handle_download_invalid_file_name(self): - task_id = 123333 - args = [str(task_id)] - self.session.getTaskInfo.return_value = {'id': task_id, - 'method': 'buildArch', - 'arch': 'taskarch', - 'state': 2} + args = [str(self.parent_task_id)] + self.session.getTaskInfo.return_value = self.parent_task_info self.list_task_output_all_volumes.return_value = {'somerpm..src.rpm': ['DEFAULT', 'vol1']} # Run it and check immediate output # args: task_id # expected: failure - with self.assertRaises(SystemExit) as ex: - anon_handle_download_task(self.options, self.session, args) - self.assertExitCode(ex, 1) - actual = self.stdout.getvalue() - expected = '' - self.assertMultiLineEqual(actual, expected) - actual = self.stderr.getvalue() - expected = 'Invalid file name: somerpm..src.rpm\n' - self.assertMultiLineEqual(actual, expected) + self.assert_system_exit( + anon_handle_download_task, + self.options, self.session, args, + stderr="Invalid file name: somerpm..src.rpm\n", + stdout='', + activate_session=None, + exit_code=1) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) - self.session.getTaskInfo.assert_called_once_with(task_id) - self.session.getTaskChildren.assert_not_called() - self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.list_task_output_all_volumes.assert_called_once_with(self.session, + self.parent_task_id) self.download_file.assert_not_called() def test_handle_download_help(self): @@ -336,7 +326,7 @@ class TestDownloadTask(utils.CliTestCase): Options: -h, --help show this help message and exit --arch=ARCH Only download packages for this arch (may be used multiple - times) + times), only for build and buildArch task methods --logs Also download build logs --topurl=URL URL under which Koji files are accessible --noprogress Do not display progress meter @@ -344,6 +334,9 @@ Options: background --nowait Do not wait for running tasks to finish -q, --quiet Suppress output + --all Download all files, all methods instead of build and buildArch + --dirpertask Download files to dir per task + --parentonly Download parent's files only """ % progname self.assertMultiLineEqual(actual, expected) actual = self.stderr.getvalue() @@ -355,35 +348,373 @@ Options: # Run it and check immediate output # no args # expected: failure - with self.assertRaises(SystemExit) as ex: - anon_handle_download_task(self.options, self.session, args) - self.assertExitCode(ex, 2) - actual = self.stdout.getvalue() - expected = '' - self.assertMultiLineEqual(actual, expected) - actual = self.stderr.getvalue() - expected = """Usage: %s download-task -(Specify the --help global option for a list of other help options) - -%s: error: Please specify a task ID -""" % (progname, progname) - self.assertEqual(actual, expected) + self.assert_system_exit( + anon_handle_download_task, + self.options, self.session, args, + stderr=self.format_error_message("Please specify a task ID"), + stdout='', + activate_session=None, + exit_code=2) def test_handle_download_multi_task_id(self): args = ["123", "456"] # Run it and check immediate output # args: 123 456 # expected: failure - with self.assertRaises(SystemExit) as ex: - anon_handle_download_task(self.options, self.session, args) - self.assertExitCode(ex, 2) + self.assert_system_exit( + anon_handle_download_task, + self.options, self.session, args, + stderr=self.format_error_message("Only one task ID may be specified"), + stdout='', + activate_session=None, + exit_code=2) + + def test_handle_download_task_parent_dirpertask(self): + args = [str(self.parent_task_id), '--arch=noarch,x86_64', '--dirpertask', '--log'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [{'id': 22222, + 'method': 'buildArch', + 'arch': 'noarch', + 'state': 2}, + {'id': 33333, + 'method': 'buildArch', + 'arch': 'x86_64', + 'state': 2}, + {'id': 44444, + 'method': 'buildArch', + 'arch': 's390', + 'state': 2}, + {'id': 55555, + 'method': 'tagBuild', + 'arch': 'noarch', + 'state': 2} + ] + self.list_task_output_all_volumes.side_effect = [ + {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']}, + {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']}, + {'somerpm.noarch.rpm': ['vol3'], + 'somelog.log': ['DEFAULT', 'vol1']}, + ] + # Run it and check immediate output + # args: task_id --arch=noarch,x86_64 --dirpertask --log + # expected: success + rv = anon_handle_download_task(self.options, self.session, args) + + actual = self.stdout.getvalue() + expected = '' + self.assertMultiLineEqual(actual, expected) + # Finally, assert that things were called as we expected. + self.ensure_connection.assert_called_once_with(self.session, self.options) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ + call(self.session, 22222), + call(self.session, 33333), + call(self.session, 55555)]) + self.assertListEqual(self.download_file.mock_calls, [ + call('https://topurl/work/tasks/2222/22222/somerpm.noarch.rpm', + '22222/somerpm.noarch.rpm', quiet=None, noprogress=None, size=5, num=1), + call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm', + '33333/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=5, num=2), + call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm', + '33333/vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=5, num=3), + call('https://topurl/work/tasks/5555/55555/somelog.log', + '55555/somelog.log', quiet=None, noprogress=None, size=5, num=4), + call('https://topurl/vol/vol1/work/tasks/5555/55555/somelog.log', + '55555/vol1/somelog.log', quiet=None, noprogress=None, size=5, num=5), + ]) + self.assertIsNone(rv) + + def test_handle_download_task_parent_all(self): + args = [str(self.parent_task_id), '--arch=noarch,x86_64', '--all'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [{'id': 22222, + 'method': 'buildArch', + 'arch': 'noarch', + 'state': 2}, + {'id': 33333, + 'method': 'buildArch', + 'arch': 'x86_64', + 'state': 2}, + {'id': 44444, + 'method': 'buildArch', + 'arch': 's390', + 'state': 2}, + {'id': 55555, + 'method': 'tagBuild', + 'arch': 'noarch', + 'state': 2} + ] + self.list_task_output_all_volumes.side_effect = [ + {'somerpm.src.rpm': ['DEFAULT', 'vol1']}, + {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']}, + {'somerpm.noarch.rpm': ['vol3'], + 'somelog.log': ['DEFAULT', 'vol1']}, + ] + # Run it and check immediate output + # args: task_id --arch=noarch,x86_64 --all + # expected: success + rv = anon_handle_download_task(self.options, self.session, args) + actual = self.stdout.getvalue() expected = '' self.assertMultiLineEqual(actual, expected) + # Finally, assert that things were called as we expected. + self.ensure_connection.assert_called_once_with(self.session, self.options) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ + call(self.session, 22222), + call(self.session, 33333), + call(self.session, 55555)]) + self.assertListEqual(self.download_file.mock_calls, [ + call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm', + 'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=1), + call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm', + 'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=2), + call('https://topurl/vol/vol3/work/tasks/5555/55555/somerpm.noarch.rpm', + 'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=3, num=3), + ]) + self.assertIsNone(rv) + + def test_handle_download_task_parent_only(self): + args = [str(self.parent_task_id), '--parentonly'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.list_task_output_all_volumes.return_value = {} + # Run it and check immediate output + # args: task_id --parentonly + # expected: pass + anon_handle_download_task(self.options, self.session, args) + actual = self.stdout.getvalue() + expected = 'No files for download found.\n' + self.assertMultiLineEqual(actual, expected) actual = self.stderr.getvalue() - expected = """Usage: %s download-task -(Specify the --help global option for a list of other help options) + expected = '' + self.assertMultiLineEqual(actual, expected) + # Finally, assert that things were called as we expected. + self.ensure_connection.assert_called_once_with(self.session, self.options) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_not_called() + self.list_task_output_all_volumes.assert_called_once_with(self.session, + self.parent_task_id) + self.download_file.assert_not_called() -%s: error: Only one task ID may be specified -""" % (progname, progname) - self.assertEqual(actual, expected) + def test_handle_download_task_parent_dirpertask_all(self): + args = [str(self.parent_task_id), '--dirpertask', '--all'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [{'id': 22222, + 'method': 'buildArch', + 'arch': 'noarch', + 'state': 2}, + {'id': 33333, + 'method': 'buildArch', + 'arch': 'x86_64', + 'state': 2}, + {'id': 44444, + 'method': 'buildArch', + 'arch': 's390', + 'state': 2}, + {'id': 55555, + 'method': 'tagBuild', + 'arch': 'noarch', + 'state': 2} + ] + self.list_task_output_all_volumes.side_effect = [ + {}, + {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']}, + {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']}, + {'somerpm.s390.rpm': ['vol2']}, + {'somerpm.noarch.rpm': ['vol3'], + 'somelog.log': ['DEFAULT', 'vol1']}, + ] + # Run it and check immediate output + # args: task_id --dirpertask --log + # expected: success + rv = anon_handle_download_task(self.options, self.session, args) + + actual = self.stdout.getvalue() + expected = '' + self.assertMultiLineEqual(actual, expected) + # Finally, assert that things were called as we expected. + self.ensure_connection.assert_called_once_with(self.session, self.options) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ + call(self.session, self.parent_task_id), + call(self.session, 22222), + call(self.session, 33333), + call(self.session, 44444), + call(self.session, 55555)]) + self.assertListEqual(self.download_file.mock_calls, [ + call('https://topurl/work/tasks/2222/22222/somerpm.src.rpm', + '22222/somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=1), + call('https://topurl/vol/vol1/work/tasks/2222/22222/somerpm.src.rpm', + '22222/vol1/somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=2), + call('https://topurl/work/tasks/2222/22222/somerpm.noarch.rpm', + '22222/somerpm.noarch.rpm', quiet=None, noprogress=None, size=7, num=3), + call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm', + '33333/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=4), + call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm', + '33333/vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=5), + call('https://topurl/vol/vol2/work/tasks/4444/44444/somerpm.s390.rpm', + '44444/vol2/somerpm.s390.rpm', quiet=None, noprogress=None, size=7, num=6), + call('https://topurl/vol/vol3/work/tasks/5555/55555/somerpm.noarch.rpm', + '55555/vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=7, num=7), + ]) + self.assertIsNone(rv) + + def test_handle_download_task_log_with_arch(self): + args = [str(self.parent_task_id), '--arch=noarch', '--log'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [{'id': 22222, + 'method': 'buildArch', + 'arch': 'noarch', + 'state': 2}] + self.list_task_output_all_volumes.side_effect = [{ + 'somerpm.src.rpm': ['DEFAULT', 'vol1'], + 'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'], + 'somerpm.noarch.rpm': ['vol3'], + 'somelog.noarch.log': ['DEFAULT', 'vol1']}] + + # Run it and check immediate output + # args: task_id --log --arch=noarch + # expected: success + rv = anon_handle_download_task(self.options, self.session, args) + + actual = self.stdout.getvalue() + expected = '' + self.assertMultiLineEqual(actual, expected) + # Finally, assert that things were called as we expected. + self.ensure_connection.assert_called_once_with(self.session, self.options) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.list_task_output_all_volumes.assert_has_calls([mock.call(self.session, 22222)]) + self.assertListEqual(self.download_file.mock_calls, [ + call('https://topurl/vol/vol3/work/tasks/2222/22222/somerpm.noarch.rpm', + 'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=3, num=1), + call('https://topurl/work/tasks/2222/22222/somelog.noarch.log', + 'somelog.noarch.log', quiet=None, noprogress=None, size=3, num=2), + call('https://topurl/vol/vol1/work/tasks/2222/22222/somelog.noarch.log', + 'vol1/somelog.noarch.log', quiet=None, noprogress=None, size=3, num=3) + ]) + self.assertIsNone(rv) + + def test_handle_download_task_all_log(self): + args = [str(self.parent_task_id), '--all', '--log'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [{'id': 22222, + 'method': 'buildArch', + 'arch': 'noarch', + 'state': 2}, + {'id': 33333, + 'method': 'buildArch', + 'arch': 'x86_64', + 'state': 2}, + {'id': 44444, + 'method': 'buildArch', + 'arch': 's390', + 'state': 2}, + {'id': 55555, + 'method': 'tagBuild', + 'arch': 'noarch', + 'state': 2} + ] + self.list_task_output_all_volumes.side_effect = [ + {}, + {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']}, + {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']}, + {'somerpm.s390.rpm': ['vol2']}, + {'somelog.log': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['vol3']}, + ] + # Run it and check immediate output + # args: task_id --all --log + # expected: success + rv = anon_handle_download_task(self.options, self.session, args) + + actual = self.stdout.getvalue() + expected = '' + self.assertMultiLineEqual(actual, expected) + # Finally, assert that things were called as we expected. + self.ensure_connection.assert_called_once_with(self.session, self.options) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ + call(self.session, self.parent_task_id), + call(self.session, 22222), + call(self.session, 33333), + call(self.session, 44444), + call(self.session, 55555)]) + self.assertListEqual(self.download_file.mock_calls, [ + call('https://topurl/work/tasks/2222/22222/somerpm.src.rpm', + 'somerpm.src.rpm', quiet=None, noprogress=None, size=9, num=1), + call('https://topurl/vol/vol1/work/tasks/2222/22222/somerpm.src.rpm', + 'vol1/somerpm.src.rpm', quiet=None, noprogress=None, size=9, num=2), + call('https://topurl/work/tasks/2222/22222/somerpm.noarch.rpm', + 'somerpm.noarch.rpm', quiet=None, noprogress=None, size=9, num=3), + call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm', + 'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=9, num=4), + call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm', + 'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=9, num=5), + call('https://topurl/vol/vol2/work/tasks/4444/44444/somerpm.s390.rpm', + 'vol2/somerpm.s390.rpm', quiet=None, noprogress=None, size=9, num=6), + call('https://topurl/work/tasks/5555/55555/somelog.log', + 'somelog.noarch.log', quiet=None, noprogress=None, size=9, num=7), + call('https://topurl/vol/vol1/work/tasks/5555/55555/somelog.log', + 'vol1/somelog.noarch.log', quiet=None, noprogress=None, size=9, num=8), + call('https://topurl/vol/vol3/work/tasks/5555/55555/somerpm.noarch.rpm', + 'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=9, num=9), + ]) + self.assertIsNone(rv) + + def test_handle_download_task_parent_dirpertask_all_conflict_names(self): + args = [str(self.parent_task_id), '--all'] + self.session.getTaskInfo.return_value = self.parent_task_info + self.session.getTaskChildren.return_value = [{'id': 22222, + 'method': 'buildArch', + 'arch': 'noarch', + 'state': 2}, + {'id': 33333, + 'method': 'buildArch', + 'arch': 'x86_64', + 'state': 2}, + {'id': 44444, + 'method': 'buildArch', + 'arch': 's390', + 'state': 2}, + {'id': 55555, + 'method': 'tagBuild', + 'arch': 'noarch', + 'state': 2} + ] + self.list_task_output_all_volumes.side_effect = [ + {}, + {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']}, + {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']}, + {'somerpm.s390.rpm': ['vol2']}, + {'somerpm.noarch.rpm': ['DEFAULT'], + 'somelog.log': ['DEFAULT', 'vol1']}, + ] + # Run it and check immediate output + # args: task_id --dirpertask --log + # expected: failure + self.assert_system_exit( + anon_handle_download_task, + self.options, self.session, args, + stderr="Download files names conflict, use --dirpertask\n", + stdout='', + activate_session=None, + exit_code=1) + actual = self.stdout.getvalue() + expected = '' + self.assertMultiLineEqual(actual, expected) + # Finally, assert that things were called as we expected. + self.ensure_connection.assert_called_once_with(self.session, self.options) + self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) + self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) + self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ + call(self.session, self.parent_task_id), + call(self.session, 22222), + call(self.session, 33333), + call(self.session, 44444), + call(self.session, 55555)]) + self.assertListEqual(self.download_file.mock_calls, [])