#3343 Download output for all type of task in download-task
Merged 2 months ago by tkopecek. Opened 4 months ago by jcupova.
jcupova/koji issue-3182  into  master

file modified
+83 -28
@@ -7325,7 +7325,8 @@ 

      usage = "usage: %prog download-task <task_id>"

      parser = OptionParser(usage=get_usage_str(usage))

      parser.add_option("--arch", dest="arches", metavar="ARCH", action="append", default=[],

-                       help="Only download packages for this arch (may be used multiple times)")

+                       help="Only download packages for this arch (may be used multiple times), "

+                            "only for build and buildArch task methods")

      parser.add_option("--logs", dest="logs", action="store_true", default=False,

                        help="Also download build logs")

      parser.add_option("--topurl", metavar="URL", default=options.topurl,
@@ -7337,6 +7338,10 @@ 

                        help="Do not wait for running tasks to finish")

      parser.add_option("-q", "--quiet", action="store_true",

                        help="Suppress output", default=options.quiet)

+     parser.add_option("--all", action="store_true",

+                       help="Download all files, all methods instead of build and buildArch")

+     parser.add_option("--dirpertask", action="store_true", help="Download files to dir per task")

+     parser.add_option("--parentonly", action="store_true", help="Download parent's files only")

  

      (suboptions, args) = parser.parse_args(args)

      if len(args) == 0:
@@ -7364,40 +7369,78 @@ 

          watch_tasks(session, [base_task_id], quiet=suboptions.quiet,

                      poll_interval=options.poll_interval, topurl=options.topurl)

  

-     def check_downloadable(task):

-         return task["method"] == "buildArch"

- 

-     downloadable_tasks = []

+     list_tasks = [base_task]

+     if not suboptions.parentonly:

+         list_tasks.extend(session.getTaskChildren(base_task_id))

  

-     if check_downloadable(base_task):

-         downloadable_tasks.append(base_task)

-     else:

-         subtasks = session.getTaskChildren(base_task_id)

-         downloadable_tasks.extend(list(filter(check_downloadable, subtasks)))

+     # support file types

+     expected_types = ['rpm', 'log']

+     for type in session.getArchiveTypes():

+         expected_types.extend(type['extensions'].split(' '))

  

      # get files for download

      downloads = []

- 

-     for task in downloadable_tasks:

-         files = list_task_output_all_volumes(session, task["id"])

-         for filename in files:

-             if filename.endswith(".rpm"):

-                 for volume in files[filename]:

-                     filearch = filename.split(".")[-2]

-                     if len(suboptions.arches) == 0 or filearch in suboptions.arches:

-                         downloads.append((task, filename, volume, filename))

-             elif filename.endswith(".log") and suboptions.logs:

-                 for volume in files[filename]:

-                     # rename logs, they would conflict

-                     new_filename = "%s.%s.log" % (filename.rstrip(".log"), task["arch"])

-                     downloads.append((task, filename, volume, new_filename))

+     build_methods_list = ['buildArch', 'build']

+     for task in list_tasks:

+         taskarch = task['arch']

+         task_id = str(task['id'])

+         if len(suboptions.arches) == 0 or taskarch in suboptions.arches:

+             files = list_task_output_all_volumes(session, task["id"])

+             filetype = None

+             for filename in files:

+                 if filename.endswith('src.rpm'):

+                     filetype = 'src.rpm'

+                 else:

+                     for ft in expected_types:

+                         if filename.endswith('.%s' % ft):

+                             filetype = ft

+                             break

+                 if not filetype:

+                     warn('Unsupported file type for download-task: %s' % filename)

+                 else:

+                     if suboptions.all and task['method'] not in build_methods_list:

+                         if filetype != 'log':

+                             for volume in files[filename]:

+                                 if suboptions.dirpertask:

+                                     new_filename = '%s/%s' % (task_id, filename)

+                                 else:

+                                     if taskarch not in filename and filetype != 'src.rpm':

+                                         part_filename = filename[:-len('.%s' % filetype)]

+                                         new_filename = "%s.%s.%s" % (part_filename,

+                                                                      taskarch, filetype)

+                                     else:

+                                         new_filename = filename

+                                 downloads.append((task, filename, volume, new_filename, task_id))

+                     elif task['method'] in build_methods_list:

+                         if filetype in ['rpm', 'src.rpm']:

+                             filearch = filename.split(".")[-2]

+                             for volume in files[filename]:

+                                 if len(suboptions.arches) == 0 or filearch in suboptions.arches:

+                                     if suboptions.dirpertask:

+                                         new_filename = '%s/%s' % (task_id, filename)

+                                     else:

+                                         new_filename = filename

+                                     downloads.append((task, filename, volume, new_filename,

+                                                       task_id))

+ 

+                     if filetype == 'log' and suboptions.logs:

+                         for volume in files[filename]:

+                             if suboptions.dirpertask:

+                                 new_filename = '%s/%s' % (task_id, filename)

+                             else:

+                                 if taskarch not in filename:

+                                     part_filename = filename[:-len('.log')]

+                                     new_filename = "%s.%s.log" % (part_filename, taskarch)

+                                 else:

+                                     new_filename = filename

+                             downloads.append((task, filename, volume, new_filename, task_id))

  

      if len(downloads) == 0:

          print("No files for download found.")

          return

  

      required_tasks = {}

-     for (task, nop, nop, nop) in downloads:

+     for (task, nop, nop, nop, nop) in downloads:

          if task["id"] not in required_tasks:

              required_tasks[task["id"]] = task

  
@@ -7408,14 +7451,26 @@ 

              else:

                  error("Child task %d has not finished yet." % task_id)

  

+     downloads_new_names = [(new_filename, vol) for (_, _, vol, new_filename, _) in downloads]

+     if not suboptions.dirpertask:

+         not_uniques = list({x for x in downloads_new_names if downloads_new_names.count(x) > 1})

+         if not_uniques:

+             error("Download files names conflict, use --dirpertask")

+ 

      # perform the download

      number = 0

      pathinfo = koji.PathInfo(topdir=suboptions.topurl)

-     for (task, filename, volume, new_filename) in downloads:

+     for (task, filename, volume, new_filename, task_id) in downloads:

+         if suboptions.dirpertask:

+             koji.ensuredir(task_id)

          number += 1

          if volume not in (None, 'DEFAULT'):

-             koji.ensuredir(volume)

-             new_filename = os.path.join(volume, new_filename)

+             if suboptions.dirpertask:

+                 koji.ensuredir('%s/%s' % (task_id, volume))

+                 new_filename = os.path.join(task_id, volume, filename)

+             else:

+                 koji.ensuredir(volume)

+                 new_filename = os.path.join(volume, new_filename)

          if '..' in filename:

              error('Invalid file name: %s' % filename)

          url = '%s/%s/%s' % (pathinfo.work(volume), pathinfo.taskrelpath(task["id"]), filename)

@@ -10,6 +10,7 @@ 

  

  class TestDownloadBuild(utils.CliTestCase):

      def setUp(self):

+         self.maxDiff = None

          self.options = mock.MagicMock()

          self.options.debug = False

          self.session = mock.MagicMock()
@@ -186,6 +187,48 @@ 

              exit_code=1

          )

  

+     def test_download_build_latest_from_error_find_build(self):

+         build_id = 'package-name'

+         self.session.listTagged.side_effect = koji.GenericError

+         self.assert_system_exit(

+             anon_handle_download_build,

+             self.options,

+             self.session,

+             [build_id, '--latestfrom', self.tag],

+             stderr='Error finding latest build: {}\n',

+             activate_session=None,

+             exit_code=1

+         )

+ 

+     def test_download_build_topurl_none(self):

+         build_id = '1'

+         self.assert_system_exit(

+             anon_handle_download_build,

+             self.options,

+             self.session,

+             [build_id, '--topurl', None],

+             stderr='You must specify --topurl to download files\n',

+             activate_session=None,

+             exit_code=1

+         )

+ 

+     @mock.patch('koji.buildLabel')

+     def test_download_build_type_not_scratch(self, build_label):

+         build_id = '1'

+         type = 'rpm'

+         self.session.listArchives.return_value = []

+         nvr = self.build_templ['nvr']

+         build_label.return_value = nvr

+         self.assert_system_exit(

+             anon_handle_download_build,

+             self.options,

+             self.session,

+             [build_id, '--type', type],

+             stderr='No %s archives available for %s\n' % (type, nvr),

+             activate_session=None,

+             exit_code=1

+         )

+ 

      def test_handle_add_volume_help(self):

          self.assert_help(

              anon_handle_download_build,

@@ -14,8 +14,6 @@ 

  

  

  class TestDownloadTask(utils.CliTestCase):

-     # Show long diffs in error output...

-     maxDiff = None

  

      def gen_calls(self, task_output, pattern, blacklist=[], arch=None):

  
@@ -40,6 +38,8 @@ 

          return calls

  

      def setUp(self):

+         # Show long diffs in error output...

+         self.maxDiff = None

          # Mock out the options parsed in main

          self.options = mock.MagicMock()

          self.options.quiet = None
@@ -53,17 +53,22 @@ 

          self.ensure_connection = mock.patch('koji_cli.commands.ensure_connection').start()

          self.stdout = mock.patch('sys.stdout', new_callable=six.StringIO).start()

          self.stderr = mock.patch('sys.stderr', new_callable=six.StringIO).start()

+         self.parent_task_id = 123333

+         self.parent_task_info = {'id': self.parent_task_id, 'method': 'buildArch',

+                                  'arch': 'taskarch', 'state': 2, 'parent': None}

+         self.error_format = """Usage: %s download-task <task_id>

+ (Specify the --help global option for a list of other help options)

+ 

+ %s: error: {message}

+ """ % (self.progname, self.progname)

  

      def tearDown(self):

          mock.patch.stopall()

  

      def test_handle_download_task_single(self):

-         task_id = 123333

-         args = [str(task_id)]

-         self.session.getTaskInfo.return_value = {'id': task_id,

-                                                  'method': 'buildArch',

-                                                  'arch': 'taskarch',

-                                                  'state': 2}

+         args = [str(self.parent_task_id)]

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = []

          self.list_task_output_all_volumes.return_value = {

              'somerpm.src.rpm': ['DEFAULT', 'vol1'],

              'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'],
@@ -84,41 +89,35 @@ 

          self.assertMultiLineEqual(actual, expected)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

-         self.session.getTaskChildren.assert_not_called()

-         self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.list_task_output_all_volumes.assert_called_once_with(self.session,

+                                                                   self.parent_task_id)

          self.assertListEqual(self.download_file.mock_calls, calls)

          self.assertIsNone(rv)

  

      def test_handle_download_task_not_found(self):

-         task_id = 123333

-         args = [str(task_id)]

+         args = [str(self.parent_task_id)]

          self.session.getTaskInfo.return_value = None

  

          # Run it and check immediate output

          # args: task_id

          # expected: error

-         with self.assertRaises(SystemExit) as ex:

-             anon_handle_download_task(self.options, self.session, args)

-         self.assertExitCode(ex, 1)

-         actual = self.stdout.getvalue()

-         expected = ''

-         self.assertMultiLineEqual(actual, expected)

-         actual = self.stderr.getvalue()

-         expected = 'No such task: %s\n' % task_id

-         self.assertMultiLineEqual(actual, expected)

+         self.assert_system_exit(

+             anon_handle_download_task,

+             self.options, self.session, args,

+             stderr='No such task: %s\n' % self.parent_task_id,

+             stdout='',

+             activate_session=None,

+             exit_code=1)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

          self.session.getTaskChildren.assert_not_called()

  

      def test_handle_download_task_parent(self):

-         task_id = 123333

-         args = [str(task_id), '--arch=noarch,x86_64']

-         self.session.getTaskInfo.return_value = {'id': task_id,

-                                                  'method': 'build',

-                                                  'arch': 'taskarch',

-                                                  'state': 2}

+         args = [str(self.parent_task_id), '--arch=noarch,x86_64']

+         self.session.getTaskInfo.return_value = self.parent_task_info

          self.session.getTaskChildren.return_value = [{'id': 22222,

                                                        'method': 'buildArch',

                                                        'arch': 'noarch',
@@ -140,7 +139,8 @@ 

              {'somerpm.src.rpm': ['DEFAULT', 'vol1']},

              {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']},

              {'somerpm.noarch.rpm': ['vol3'],

-              'somelog.log': ['DEFAULT', 'vol1']}]

+              'somelog.log': ['DEFAULT', 'vol1']},

+         ]

          # Run it and check immediate output

          # args: task_id --arch=noarch,x86_64

          # expected: success
@@ -151,36 +151,31 @@ 

          self.assertMultiLineEqual(actual, expected)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

-         self.session.getTaskChildren.assert_called_once_with(task_id)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

          self.assertEqual(self.list_task_output_all_volumes.mock_calls, [

              call(self.session, 22222),

              call(self.session, 33333),

-             call(self.session, 44444)])

+             call(self.session, 55555)])

          self.assertListEqual(self.download_file.mock_calls, [

              call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm',

-                  'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=1),

+                  'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=2, num=1),

              call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm',

-                  'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=2),

-             call('https://topurl/vol/vol3/work/tasks/4444/44444/somerpm.noarch.rpm',

-                  'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=3, num=3)])

+                  'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=2, num=2)])

          self.assertIsNone(rv)

  

      def test_handle_download_task_log(self):

-         task_id = 123333

-         args = [str(task_id), '--log']

-         self.session.getTaskInfo.return_value = {'id': task_id,

-                                                  'method': 'buildArch',

-                                                  'arch': 'taskarch',

-                                                  'state': 2}

-         self.list_task_output_all_volumes.return_value = {

+         args = [str(self.parent_task_id), '--log']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = [{'id': 22222,

+                                                       'method': 'buildArch',

+                                                       'arch': 'noarch',

+                                                       'state': 2}]

+         self.list_task_output_all_volumes.side_effect = [{}, {

              'somerpm.src.rpm': ['DEFAULT', 'vol1'],

              'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'],

              'somerpm.noarch.rpm': ['vol3'],

-             'somelog.log': ['DEFAULT', 'vol1']}

- 

-         calls = self.gen_calls(self.list_task_output_all_volumes.return_value,

-                                'https://topurl/%swork/tasks/3333/123333/%s', arch='taskarch')

+             'somelog.log': ['DEFAULT', 'vol1']}]

  

          # Run it and check immediate output

          # args: task_id --log
@@ -192,26 +187,31 @@ 

          self.assertMultiLineEqual(actual, expected)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

-         self.session.getTaskChildren.assert_not_called()

-         self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id)

-         self.assertListEqual(self.download_file.mock_calls, calls)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.list_task_output_all_volumes.assert_has_calls([

+             mock.call(self.session, self.parent_task_id), mock.call(self.session, 22222)])

+         self.assertListEqual(self.download_file.mock_calls, [

+             call('https://topurl/work/tasks/2222/22222/somerpm.src.rpm',

+                  'somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=1),

+             call('https://topurl/vol/vol1/work/tasks/2222/22222/somerpm.src.rpm',

+                  'vol1/somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=2),

+             call('https://topurl/work/tasks/2222/22222/somerpm.x86_64.rpm',

+                  'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=3),

+             call('https://topurl/vol/vol2/work/tasks/2222/22222/somerpm.x86_64.rpm',

+                  'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=4),

+             call('https://topurl/vol/vol3/work/tasks/2222/22222/somerpm.noarch.rpm',

+                  'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=7, num=5),

+             call('https://topurl/work/tasks/2222/22222/somelog.log',

+                  'somelog.noarch.log', quiet=None, noprogress=None, size=7, num=6),

+             call('https://topurl/vol/vol1/work/tasks/2222/22222/somelog.log',

+                  'vol1/somelog.noarch.log', quiet=None, noprogress=None, size=7, num=7)

+         ])

          self.assertIsNone(rv)

  

      def test_handle_download_no_download(self):

-         task_id = 123333

-         args = [str(task_id), '--arch=s390,ppc']

-         self.session.getTaskInfo.return_value = {'id': task_id,

-                                                  'method': 'buildArch',

-                                                  'arch': 'taskarch',

-                                                  'state': 2}

-         self.list_task_output_all_volumes.return_value = {

-             'somerpm.src.rpm': ['DEFAULT', 'vol1'],

-             'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'],

-             'somerpm.noarch.rpm': ['vol3'],

-             'somelog.log': ['DEFAULT', 'vol1'],

-             'somezip.zip': ['DEFAULT']

-         }

+         args = [str(self.parent_task_id), '--arch=s390,ppc']

+         self.session.getTaskInfo.return_value = self.parent_task_info

  

          # Run it and check immediate output

          # args: task_id --arch=s390,ppc
@@ -225,15 +225,14 @@ 

          self.assertMultiLineEqual(actual, expected)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

-         self.session.getTaskChildren.assert_not_called()

-         self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.list_task_output_all_volumes.assert_not_called()

          self.download_file.assert_not_called()

  

      def test_handle_download_parent_not_finished(self):

-         task_id = 123333

-         args = [str(task_id)]

-         self.session.getTaskInfo.return_value = {'id': task_id,

+         args = [str(self.parent_task_id)]

+         self.session.getTaskInfo.return_value = {'id': self.parent_task_id,

                                                   'method': 'buildArch',

                                                   'arch': 'taskarch',

                                                   'state': 3}
@@ -247,78 +246,69 @@ 

          # Run it and check immediate output

          # args: task_id

          # expected: failure

-         with self.assertRaises(SystemExit) as ex:

-             anon_handle_download_task(self.options, self.session, args)

-         self.assertExitCode(ex, 1)

-         actual = self.stdout.getvalue()

-         expected = ''

-         self.assertMultiLineEqual(actual, expected)

-         actual = self.stderr.getvalue()

-         expected = 'Task 123333 has not finished yet.\n'

-         self.assertMultiLineEqual(actual, expected)

+         self.assert_system_exit(

+             anon_handle_download_task,

+             self.options, self.session, args,

+             stderr="Task 123333 has not finished yet.\n",

+             stdout='',

+             activate_session=None,

+             exit_code=1)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

-         self.session.getTaskChildren.assert_not_called()

-         self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.list_task_output_all_volumes.assert_called_once_with(

+             self.session, self.parent_task_id)

          self.download_file.assert_not_called()

  

      def test_handle_download_child_not_finished(self):

-         task_id = 123333

-         args = [str(task_id)]

-         self.session.getTaskInfo.return_value = {'id': task_id,

-                                                  'method': 'build',

-                                                  'arch': 'taskarch',

-                                                  'state': 2}

+         args = [str(self.parent_task_id)]

+         self.session.getTaskInfo.return_value = self.parent_task_info

          self.session.getTaskChildren.return_value = [{'id': 22222,

                                                        'method': 'buildArch',

                                                        'arch': 'noarch',

                                                        'state': 3}]

-         self.list_task_output_all_volumes.return_value = {'somerpm.src.rpm': ['DEFAULT', 'vol1']}

+         self.list_task_output_all_volumes.side_effect = [

+             {'somerpm.src.rpm': ['DEFAULT', 'vol1']},

+             {'somenextrpm.src.rpm': ['DEFAULT', 'vol1']}]

          # Run it and check immediate output

          # args: task_id

          # expected: failure

-         with self.assertRaises(SystemExit) as ex:

-             anon_handle_download_task(self.options, self.session, args)

-         self.assertExitCode(ex, 1)

-         actual = self.stdout.getvalue()

-         expected = ''

-         self.assertMultiLineEqual(actual, expected)

-         actual = self.stderr.getvalue()

-         expected = 'Child task 22222 has not finished yet.\n'

-         self.assertMultiLineEqual(actual, expected)

+         self.assert_system_exit(

+             anon_handle_download_task,

+             self.options, self.session, args,

+             stderr="Child task 22222 has not finished yet.\n",

+             stdout='',

+             activate_session=None,

+             exit_code=1)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

-         self.session.getTaskChildren.assert_called_once_with(task_id)

-         self.list_task_output_all_volumes.assert_called_once_with(self.session, 22222)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.list_task_output_all_volumes.assert_has_calls(

+             [mock.call(self.session, self.parent_task_id), mock.call(self.session, 22222)])

          self.download_file.assert_not_called()

  

      def test_handle_download_invalid_file_name(self):

-         task_id = 123333

-         args = [str(task_id)]

-         self.session.getTaskInfo.return_value = {'id': task_id,

-                                                  'method': 'buildArch',

-                                                  'arch': 'taskarch',

-                                                  'state': 2}

+         args = [str(self.parent_task_id)]

+         self.session.getTaskInfo.return_value = self.parent_task_info

          self.list_task_output_all_volumes.return_value = {'somerpm..src.rpm': ['DEFAULT', 'vol1']}

          # Run it and check immediate output

          # args: task_id

          # expected: failure

-         with self.assertRaises(SystemExit) as ex:

-             anon_handle_download_task(self.options, self.session, args)

-         self.assertExitCode(ex, 1)

-         actual = self.stdout.getvalue()

-         expected = ''

-         self.assertMultiLineEqual(actual, expected)

-         actual = self.stderr.getvalue()

-         expected = 'Invalid file name: somerpm..src.rpm\n'

-         self.assertMultiLineEqual(actual, expected)

+         self.assert_system_exit(

+             anon_handle_download_task,

+             self.options, self.session, args,

+             stderr="Invalid file name: somerpm..src.rpm\n",

+             stdout='',

+             activate_session=None,

+             exit_code=1)

          # Finally, assert that things were called as we expected.

          self.ensure_connection.assert_called_once_with(self.session, self.options)

-         self.session.getTaskInfo.assert_called_once_with(task_id)

-         self.session.getTaskChildren.assert_not_called()

-         self.list_task_output_all_volumes.assert_called_once_with(self.session, task_id)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.list_task_output_all_volumes.assert_called_once_with(self.session,

+                                                                   self.parent_task_id)

          self.download_file.assert_not_called()

  

      def test_handle_download_help(self):
@@ -336,7 +326,7 @@ 

  Options:

    -h, --help    show this help message and exit

    --arch=ARCH   Only download packages for this arch (may be used multiple

-                 times)

+                 times), only for build and buildArch task methods

    --logs        Also download build logs

    --topurl=URL  URL under which Koji files are accessible

    --noprogress  Do not display progress meter
@@ -344,6 +334,9 @@ 

                  background

    --nowait      Do not wait for running tasks to finish

    -q, --quiet   Suppress output

+   --all         Download all files, all methods instead of build and buildArch

+   --dirpertask  Download files to dir per task

+   --parentonly  Download parent's files only

  """ % progname

          self.assertMultiLineEqual(actual, expected)

          actual = self.stderr.getvalue()
@@ -355,35 +348,373 @@ 

          # Run it and check immediate output

          # no args

          # expected: failure

-         with self.assertRaises(SystemExit) as ex:

-             anon_handle_download_task(self.options, self.session, args)

-         self.assertExitCode(ex, 2)

-         actual = self.stdout.getvalue()

-         expected = ''

-         self.assertMultiLineEqual(actual, expected)

-         actual = self.stderr.getvalue()

-         expected = """Usage: %s download-task <task_id>

- (Specify the --help global option for a list of other help options)

- 

- %s: error: Please specify a task ID

- """ % (progname, progname)

-         self.assertEqual(actual, expected)

+         self.assert_system_exit(

+             anon_handle_download_task,

+             self.options, self.session, args,

+             stderr=self.format_error_message("Please specify a task ID"),

+             stdout='',

+             activate_session=None,

+             exit_code=2)

  

      def test_handle_download_multi_task_id(self):

          args = ["123", "456"]

          # Run it and check immediate output

          # args: 123 456

          # expected: failure

-         with self.assertRaises(SystemExit) as ex:

-             anon_handle_download_task(self.options, self.session, args)

-         self.assertExitCode(ex, 2)

+         self.assert_system_exit(

+             anon_handle_download_task,

+             self.options, self.session, args,

+             stderr=self.format_error_message("Only one task ID may be specified"),

+             stdout='',

+             activate_session=None,

+             exit_code=2)

+ 

+     def test_handle_download_task_parent_dirpertask(self):

+         args = [str(self.parent_task_id), '--arch=noarch,x86_64', '--dirpertask', '--log']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = [{'id': 22222,

+                                                       'method': 'buildArch',

+                                                       'arch': 'noarch',

+                                                       'state': 2},

+                                                      {'id': 33333,

+                                                       'method': 'buildArch',

+                                                       'arch': 'x86_64',

+                                                       'state': 2},

+                                                      {'id': 44444,

+                                                       'method': 'buildArch',

+                                                       'arch': 's390',

+                                                       'state': 2},

+                                                      {'id': 55555,

+                                                       'method': 'tagBuild',

+                                                       'arch': 'noarch',

+                                                       'state': 2}

+                                                      ]

+         self.list_task_output_all_volumes.side_effect = [

+             {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']},

+             {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']},

+             {'somerpm.noarch.rpm': ['vol3'],

+              'somelog.log': ['DEFAULT', 'vol1']},

+         ]

+         # Run it and check immediate output

+         # args: task_id --arch=noarch,x86_64 --dirpertask --log

+         # expected: success

+         rv = anon_handle_download_task(self.options, self.session, args)

+ 

+         actual = self.stdout.getvalue()

+         expected = ''

+         self.assertMultiLineEqual(actual, expected)

+         # Finally, assert that things were called as we expected.

+         self.ensure_connection.assert_called_once_with(self.session, self.options)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.assertEqual(self.list_task_output_all_volumes.mock_calls, [

+             call(self.session, 22222),

+             call(self.session, 33333),

+             call(self.session, 55555)])

+         self.assertListEqual(self.download_file.mock_calls, [

+             call('https://topurl/work/tasks/2222/22222/somerpm.noarch.rpm',

+                  '22222/somerpm.noarch.rpm', quiet=None, noprogress=None, size=5, num=1),

+             call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  '33333/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=5, num=2),

+             call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  '33333/vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=5, num=3),

+             call('https://topurl/work/tasks/5555/55555/somelog.log',

+                  '55555/somelog.log', quiet=None, noprogress=None, size=5, num=4),

+             call('https://topurl/vol/vol1/work/tasks/5555/55555/somelog.log',

+                  '55555/vol1/somelog.log', quiet=None, noprogress=None, size=5, num=5),

+         ])

+         self.assertIsNone(rv)

+ 

+     def test_handle_download_task_parent_all(self):

+         args = [str(self.parent_task_id), '--arch=noarch,x86_64', '--all']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = [{'id': 22222,

+                                                       'method': 'buildArch',

+                                                       'arch': 'noarch',

+                                                       'state': 2},

+                                                      {'id': 33333,

+                                                       'method': 'buildArch',

+                                                       'arch': 'x86_64',

+                                                       'state': 2},

+                                                      {'id': 44444,

+                                                       'method': 'buildArch',

+                                                       'arch': 's390',

+                                                       'state': 2},

+                                                      {'id': 55555,

+                                                       'method': 'tagBuild',

+                                                       'arch': 'noarch',

+                                                       'state': 2}

+                                                      ]

+         self.list_task_output_all_volumes.side_effect = [

+             {'somerpm.src.rpm': ['DEFAULT', 'vol1']},

+             {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']},

+             {'somerpm.noarch.rpm': ['vol3'],

+              'somelog.log': ['DEFAULT', 'vol1']},

+         ]

+         # Run it and check immediate output

+         # args: task_id --arch=noarch,x86_64 --all

+         # expected: success

+         rv = anon_handle_download_task(self.options, self.session, args)

+ 

          actual = self.stdout.getvalue()

          expected = ''

          self.assertMultiLineEqual(actual, expected)

+         # Finally, assert that things were called as we expected.

+         self.ensure_connection.assert_called_once_with(self.session, self.options)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.assertEqual(self.list_task_output_all_volumes.mock_calls, [

+             call(self.session, 22222),

+             call(self.session, 33333),

+             call(self.session, 55555)])

+         self.assertListEqual(self.download_file.mock_calls, [

+             call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=1),

+             call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=3, num=2),

+             call('https://topurl/vol/vol3/work/tasks/5555/55555/somerpm.noarch.rpm',

+                  'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=3, num=3),

+         ])

+         self.assertIsNone(rv)

+ 

+     def test_handle_download_task_parent_only(self):

+         args = [str(self.parent_task_id), '--parentonly']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.list_task_output_all_volumes.return_value = {}

+         # Run it and check immediate output

+         # args: task_id --parentonly

+         # expected: pass

+         anon_handle_download_task(self.options, self.session, args)

+         actual = self.stdout.getvalue()

+         expected = 'No files for download found.\n'

+         self.assertMultiLineEqual(actual, expected)

          actual = self.stderr.getvalue()

-         expected = """Usage: %s download-task <task_id>

- (Specify the --help global option for a list of other help options)

+         expected = ''

+         self.assertMultiLineEqual(actual, expected)

+         # Finally, assert that things were called as we expected.

+         self.ensure_connection.assert_called_once_with(self.session, self.options)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_not_called()

+         self.list_task_output_all_volumes.assert_called_once_with(self.session,

+                                                                   self.parent_task_id)

+         self.download_file.assert_not_called()

  

- %s: error: Only one task ID may be specified

- """ % (progname, progname)

-         self.assertEqual(actual, expected)

+     def test_handle_download_task_parent_dirpertask_all(self):

+         args = [str(self.parent_task_id), '--dirpertask', '--all']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = [{'id': 22222,

+                                                       'method': 'buildArch',

+                                                       'arch': 'noarch',

+                                                       'state': 2},

+                                                      {'id': 33333,

+                                                       'method': 'buildArch',

+                                                       'arch': 'x86_64',

+                                                       'state': 2},

+                                                      {'id': 44444,

+                                                       'method': 'buildArch',

+                                                       'arch': 's390',

+                                                       'state': 2},

+                                                      {'id': 55555,

+                                                       'method': 'tagBuild',

+                                                       'arch': 'noarch',

+                                                       'state': 2}

+                                                      ]

+         self.list_task_output_all_volumes.side_effect = [

+             {},

+             {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']},

+             {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']},

+             {'somerpm.s390.rpm': ['vol2']},

+             {'somerpm.noarch.rpm': ['vol3'],

+              'somelog.log': ['DEFAULT', 'vol1']},

+         ]

+         # Run it and check immediate output

+         # args: task_id --dirpertask --log

+         # expected: success

+         rv = anon_handle_download_task(self.options, self.session, args)

+ 

+         actual = self.stdout.getvalue()

+         expected = ''

+         self.assertMultiLineEqual(actual, expected)

+         # Finally, assert that things were called as we expected.

+         self.ensure_connection.assert_called_once_with(self.session, self.options)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.assertEqual(self.list_task_output_all_volumes.mock_calls, [

+             call(self.session, self.parent_task_id),

+             call(self.session, 22222),

+             call(self.session, 33333),

+             call(self.session, 44444),

+             call(self.session, 55555)])

+         self.assertListEqual(self.download_file.mock_calls, [

+             call('https://topurl/work/tasks/2222/22222/somerpm.src.rpm',

+                  '22222/somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=1),

+             call('https://topurl/vol/vol1/work/tasks/2222/22222/somerpm.src.rpm',

+                  '22222/vol1/somerpm.src.rpm', quiet=None, noprogress=None, size=7, num=2),

+             call('https://topurl/work/tasks/2222/22222/somerpm.noarch.rpm',

+                  '22222/somerpm.noarch.rpm', quiet=None, noprogress=None, size=7, num=3),

+             call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  '33333/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=4),

+             call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  '33333/vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=7, num=5),

+             call('https://topurl/vol/vol2/work/tasks/4444/44444/somerpm.s390.rpm',

+                  '44444/vol2/somerpm.s390.rpm', quiet=None, noprogress=None, size=7, num=6),

+             call('https://topurl/vol/vol3/work/tasks/5555/55555/somerpm.noarch.rpm',

+                  '55555/vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=7, num=7),

+         ])

+         self.assertIsNone(rv)

+ 

+     def test_handle_download_task_log_with_arch(self):

+         args = [str(self.parent_task_id), '--arch=noarch', '--log']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = [{'id': 22222,

+                                                       'method': 'buildArch',

+                                                       'arch': 'noarch',

+                                                       'state': 2}]

+         self.list_task_output_all_volumes.side_effect = [{

+             'somerpm.src.rpm': ['DEFAULT', 'vol1'],

+             'somerpm.x86_64.rpm': ['DEFAULT', 'vol2'],

+             'somerpm.noarch.rpm': ['vol3'],

+             'somelog.noarch.log': ['DEFAULT', 'vol1']}]

+ 

+         # Run it and check immediate output

+         # args: task_id --log --arch=noarch

+         # expected: success

+         rv = anon_handle_download_task(self.options, self.session, args)

+ 

+         actual = self.stdout.getvalue()

+         expected = ''

+         self.assertMultiLineEqual(actual, expected)

+         # Finally, assert that things were called as we expected.

+         self.ensure_connection.assert_called_once_with(self.session, self.options)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.list_task_output_all_volumes.assert_has_calls([mock.call(self.session, 22222)])

+         self.assertListEqual(self.download_file.mock_calls, [

+             call('https://topurl/vol/vol3/work/tasks/2222/22222/somerpm.noarch.rpm',

+                  'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=3, num=1),

+             call('https://topurl/work/tasks/2222/22222/somelog.noarch.log',

+                  'somelog.noarch.log', quiet=None, noprogress=None, size=3, num=2),

+             call('https://topurl/vol/vol1/work/tasks/2222/22222/somelog.noarch.log',

+                  'vol1/somelog.noarch.log', quiet=None, noprogress=None, size=3, num=3)

+         ])

+         self.assertIsNone(rv)

+ 

+     def test_handle_download_task_all_log(self):

+         args = [str(self.parent_task_id), '--all', '--log']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = [{'id': 22222,

+                                                       'method': 'buildArch',

+                                                       'arch': 'noarch',

+                                                       'state': 2},

+                                                      {'id': 33333,

+                                                       'method': 'buildArch',

+                                                       'arch': 'x86_64',

+                                                       'state': 2},

+                                                      {'id': 44444,

+                                                       'method': 'buildArch',

+                                                       'arch': 's390',

+                                                       'state': 2},

+                                                      {'id': 55555,

+                                                       'method': 'tagBuild',

+                                                       'arch': 'noarch',

+                                                       'state': 2}

+                                                      ]

+         self.list_task_output_all_volumes.side_effect = [

+             {},

+             {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']},

+             {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']},

+             {'somerpm.s390.rpm': ['vol2']},

+             {'somelog.log': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['vol3']},

+         ]

+         # Run it and check immediate output

+         # args: task_id --all --log

+         # expected: success

+         rv = anon_handle_download_task(self.options, self.session, args)

+ 

+         actual = self.stdout.getvalue()

+         expected = ''

+         self.assertMultiLineEqual(actual, expected)

+         # Finally, assert that things were called as we expected.

+         self.ensure_connection.assert_called_once_with(self.session, self.options)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.assertEqual(self.list_task_output_all_volumes.mock_calls, [

+             call(self.session, self.parent_task_id),

+             call(self.session, 22222),

+             call(self.session, 33333),

+             call(self.session, 44444),

+             call(self.session, 55555)])

+         self.assertListEqual(self.download_file.mock_calls, [

+             call('https://topurl/work/tasks/2222/22222/somerpm.src.rpm',

+                  'somerpm.src.rpm', quiet=None, noprogress=None, size=9, num=1),

+             call('https://topurl/vol/vol1/work/tasks/2222/22222/somerpm.src.rpm',

+                  'vol1/somerpm.src.rpm', quiet=None, noprogress=None, size=9, num=2),

+             call('https://topurl/work/tasks/2222/22222/somerpm.noarch.rpm',

+                  'somerpm.noarch.rpm', quiet=None, noprogress=None, size=9, num=3),

+             call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=9, num=4),

+             call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm',

+                  'vol2/somerpm.x86_64.rpm', quiet=None, noprogress=None, size=9, num=5),

+             call('https://topurl/vol/vol2/work/tasks/4444/44444/somerpm.s390.rpm',

+                  'vol2/somerpm.s390.rpm', quiet=None, noprogress=None, size=9, num=6),

+             call('https://topurl/work/tasks/5555/55555/somelog.log',

+                  'somelog.noarch.log', quiet=None, noprogress=None, size=9, num=7),

+             call('https://topurl/vol/vol1/work/tasks/5555/55555/somelog.log',

+                  'vol1/somelog.noarch.log', quiet=None, noprogress=None, size=9, num=8),

+             call('https://topurl/vol/vol3/work/tasks/5555/55555/somerpm.noarch.rpm',

+                  'vol3/somerpm.noarch.rpm', quiet=None, noprogress=None, size=9, num=9),

+         ])

+         self.assertIsNone(rv)

+ 

+     def test_handle_download_task_parent_dirpertask_all_conflict_names(self):

+         args = [str(self.parent_task_id), '--all']

+         self.session.getTaskInfo.return_value = self.parent_task_info

+         self.session.getTaskChildren.return_value = [{'id': 22222,

+                                                       'method': 'buildArch',

+                                                       'arch': 'noarch',

+                                                       'state': 2},

+                                                      {'id': 33333,

+                                                       'method': 'buildArch',

+                                                       'arch': 'x86_64',

+                                                       'state': 2},

+                                                      {'id': 44444,

+                                                       'method': 'buildArch',

+                                                       'arch': 's390',

+                                                       'state': 2},

+                                                      {'id': 55555,

+                                                       'method': 'tagBuild',

+                                                       'arch': 'noarch',

+                                                       'state': 2}

+                                                      ]

+         self.list_task_output_all_volumes.side_effect = [

+             {},

+             {'somerpm.src.rpm': ['DEFAULT', 'vol1'], 'somerpm.noarch.rpm': ['DEFAULT']},

+             {'somerpm.x86_64.rpm': ['DEFAULT', 'vol2']},

+             {'somerpm.s390.rpm': ['vol2']},

+             {'somerpm.noarch.rpm': ['DEFAULT'],

+              'somelog.log': ['DEFAULT', 'vol1']},

+         ]

+         # Run it and check immediate output

+         # args: task_id --dirpertask --log

+         # expected: failure

+         self.assert_system_exit(

+             anon_handle_download_task,

+             self.options, self.session, args,

+             stderr="Download files names conflict, use --dirpertask\n",

+             stdout='',

+             activate_session=None,

+             exit_code=1)

+         actual = self.stdout.getvalue()

+         expected = ''

+         self.assertMultiLineEqual(actual, expected)

+         # Finally, assert that things were called as we expected.

+         self.ensure_connection.assert_called_once_with(self.session, self.options)

+         self.session.getTaskInfo.assert_called_once_with(self.parent_task_id)

+         self.session.getTaskChildren.assert_called_once_with(self.parent_task_id)

+         self.assertEqual(self.list_task_output_all_volumes.mock_calls, [

+             call(self.session, self.parent_task_id),

+             call(self.session, 22222),

+             call(self.session, 33333),

+             call(self.session, 44444),

+             call(self.session, 55555)])

+         self.assertListEqual(self.download_file.mock_calls, [])

rebased onto b3368f9709f5839e95ff4b35b828de11e018cd1e

3 months ago

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-ready

3 months ago

Metadata Update from @tkopecek:
- Pull-request untagged with: testing-ready

3 months ago

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-ready

3 months ago

rebased onto 153acdc671794d9b609555360c02fca71dfcb664

3 months ago

rebased onto f6031a8d05595dc5d4b9760dc69d91010a2feb28

3 months ago

Metadata Update from @jcupova:
- Pull-request untagged with: testing-ready

3 months ago

rebased onto 168dbe8726a2d9940dc7ffe983ac18484fb7d5b5

3 months ago

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-ready

3 months ago

rebased onto dd4b70ed6aa1940005d4e62a6adc23fa5c38b36c

2 months ago

rebased onto d5a28cde157355c94dcf14453fd6fcaf35c13e18

2 months ago

rebased onto 7f5eb283bc4cb5e31f42e2e1beb594ad4aefc127

2 months ago

Metadata Update from @mfilip:
- Pull-request tagged with: testing-done

2 months ago

rebased onto 8a13ffa

2 months ago

Commit 0de77b1 fixes this pull-request

Pull-Request has been merged by tkopecek

2 months ago