Commit 78007b5 Add update job scheduling and reporting

10 files Authored and Committed by adamwill a year ago
Add update job scheduling and reporting

Summary:
This adds a `jobs_from_update` function to schedule jobs for a
specified Bodhi update, a CLI subcommand for doing this, and
adjusts the scheduling fedmsg consumer to do it automatically
whenever a critpath update is created (technically, whenever a
push to 'testing' is requested, but that's almost always on
creation) or edited. It also adjusts the reporting code to
handle update test jobs: the consumer and CLI code do not need
adjustment. We can tweak this in future to run the tests for
updates with particular packages or whatever, just running for
every critpath update seems like a good place to start. I
specifically didn't get any smarter about only running the tests
if the edit changed the packages because it lets us say 'if you
want the tests to be re-run, just edit the update'.

Test Plan:
The test suite was extended, so check that :) Also,
try scheduling jobs via the CLI, and if you can, by using the
testing consumer and replaying appropriate fedmsgs (or just
turning on the prod consumer, pointing it at your openQA
instance and waiting...). Try reporting an update job result
to a local test instance of ResultsDB using the CLI. Note:
you need the very latest resultsdb_conventions release (so use
git master or grab the package from Koji).

Reviewers: jsedlak

Reviewed By: jsedlak

Subscribers: tflink

Differential Revision: https://phab.qa.fedoraproject.org/D1152

    
 1 @@ -43,10 +43,7 @@
 2   ### SUB-COMMAND METHODS
 3   
 4   def command_compose(args):
 5 -     """run OpenQA on a specified compose, optionally reporting
 6 -     results if a matching wikitcms ValidationEvent is found by
 7 -     relval/wikitcms.
 8 -     """
 9 +     """Schedule openQA jobs for a specified compose."""
10       extraparams = None
11       if args.updates:
12           extraparams = {'GRUBADD': "inst.updates={0}".format(args.updates)}
13 @@ -72,6 +69,13 @@
14       sys.exit()
15   
16   
17 + def command_update(args):
18 +     """Schedule openQA jobs for a specified update."""
19 +     jobs = schedule.jobs_from_update(args.update, args.release, flavors=[args.flavor], force=args.force,
20 +                                      openqa_hostname=args.openqa_hostname)
21 +     print("Scheduled jobs: {0}".format(', '.join((str(job) for job in jobs))))
22 +     sys.exit()
23 + 
24   def command_report(args):
25       """Map a list of openQA job IDs and/or builds to Wikitcms test
26       results, and either display the ResTups for inspection or report
27 @@ -140,24 +144,34 @@
28           "Run OpenQA tests for a release validation test event."))
29       subparsers = parser.add_subparsers()
30   
31 -     parser_compose = subparsers.add_parser(
32 -         'compose', description="Run for a specific compose (TC/RC or nightly). If a matching "
33 -         "release validation test event can be found and --submit-results is passed, results "
34 -         "will be reported.")
35 +     parser_compose = subparsers.add_parser('compose', description="Schedule jobs for a specific compose.")
36       parser_compose.add_argument(
37 -         'location', help="The top-level URL of the compose",
38 -         metavar="https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-24-20160113.n.1/compose")
39 +         'location', help="The URL of the compose (for Pungi 4 composes, the /compose directory)",
40 +         metavar="COMPOSE_URL")
41       parser_compose.add_argument(
42           "--openqa-hostname", help="openQA host to schedule jobs on (default: client library "
43 -         "default)", metavar='localhost')
44 +         "default)", metavar='HOSTNAME')
45       parser_compose.add_argument(
46           '--force', '-f', help="For each ISO/flavor combination, schedule jobs even if there "
47           "are existing, non-cancelled jobs for that combination", action='store_true')
48       parser_compose.add_argument(
49           '--updates', '-u', help="URL to an updates image to load for all tests. The tests that "
50 -         "test updates image loading will fail when you use this")
51 +         "test updates image loading will fail when you use this", metavar='UPDATE_IMAGE_URL')
52       parser_compose.set_defaults(func=command_compose)
53   
54 +     parser_update = subparsers.add_parser('update', description="Schedule jobs for a specific update.")
55 +     parser_update.add_argument('update', help="The update ID (e.g. 'FEDORA-2017-b07d628952')", metavar='UPDATE')
56 +     parser_update.add_argument('release', help="The release the update is for (e.g. '25')", type=int, metavar="NN")
57 +     parser_update.add_argument('--flavor', help="A single flavor to schedule jobs for (e.g. 'server'), "
58 +                                "otherwise jobs will be scheduled for all update flavors")
59 +     parser_update.add_argument(
60 +         "--openqa-hostname", help="openQA host to schedule jobs on (default: client library "
61 +         "default)", metavar='HOSTNAME')
62 +     parser_update.add_argument(
63 +         '--force', '-f', help="For each flavor, schedule jobs even if there "
64 +         "are existing, non-cancelled jobs for the update for that flavor", action='store_true')
65 +     parser_update.set_defaults(func=command_update)
66 + 
67       parser_report = subparsers.add_parser(
68           'report', description="Map openQA job results to Wikitcms test results and either log them to output or "
69           "submit them to the wiki and/or ResultsDB.")
70 @@ -167,17 +181,17 @@
71       parser_report.set_defaults(func=command_report)
72       parser_report.add_argument(
73           "--openqa-hostname", help="openQA host to query for results (default: client library "
74 -         "default)", metavar='localhost')
75 +         "default)", metavar='HOSTNAME')
76       parser_report.add_argument(
77           "--openqa-baseurl", help="Public openQA base URL for producing links to results (default: "
78 -         "client library baseurl)", metavar='https://openqa.example.org')
79 +         "client library baseurl, e.g. 'https://openqa.example.org')", metavar='OPENQA_BASEURL')
80       parser_report.add_argument(
81           "--wiki", action="store_true", default=False, help="Submit results to wiki")
82       parser_report.add_argument(
83           "--resultsdb", action="store_true", default=False, help="Submit results to ResultsDB")
84       parser_report.add_argument(
85           "--wiki-hostname", help="Mediawiki host to report to (default: stg.fedoraproject.org). "
86 -         "Scheme 'https' and path '/w/' are currently hard coded", metavar='fedoraproject.org')
87 +         "Scheme 'https' and path '/w/' are currently hard coded", metavar='WIKI_HOSTNAME')
88       parser_report.add_argument(
89           "--resultsdb-url", help="ResultsDB URL to report to (default: "
90           "http://localhost:5001/api/v2.0/)")
  1 @@ -91,7 +91,7 @@
  2   
  3   class OpenQAScheduler(fedmsg.consumers.FedmsgConsumer):
  4       """A fedmsg consumer that schedules openQA jobs when a new compose
  5 -     appears.
  6 +     or update appears.
  7       """
  8   
  9       def _log(self, level, message):
 10 @@ -101,15 +101,15 @@
 11           logfnc = getattr(self.log, level)
 12           logfnc("%s: %s", self.__class__.__name__, message)
 13   
 14 -     def consume(self, message):
 15 -         """Consume incoming message."""
 16 +     def _consume_compose(self, message):
 17 +         """Consume a 'compose' type message."""
 18           status = message['body']['msg'].get('status')
 19           location = message['body']['msg'].get('location')
 20           compstr = message['body']['msg'].get('compose_id', location)
 21   
 22           if 'FINISHED' in status and location:
 23               # We have a complete pungi4 compose
 24 -             self._log('info', "Scheduling openQA jobs for {0}".format(compstr))
 25 +             self._log('info', "Scheduling openQA jobs for compose {0}".format(compstr))
 26               try:
 27                   # pylint: disable=no-member
 28                   (compose, jobs) = schedule.jobs_from_compose(location, openqa_hostname=self.openqa_hostname)
 29 @@ -117,7 +117,7 @@
 30                   self._log('warning', "No openQA jobs run! {0}".format(err))
 31                   return
 32               if jobs:
 33 -                 self._log('info', "openQA jobs run on {0}: "
 34 +                 self._log('info', "openQA jobs run on compose {0}: "
 35                             "{1}".format(compose, ' '.join(str(job) for job in jobs)))
 36               else:
 37                   self._log('warning', "No openQA jobs run!")
 38 @@ -128,12 +128,41 @@
 39   
 40           return
 41   
 42 +     def _consume_update(self, message):
 43 +         """Consume an 'update' type message."""
 44 +         update = message['body']['msg'].get('update', {})
 45 +         advisory = update.get('alias')
 46 +         critpath = update.get('critpath', False)
 47 +         version = update.get('release', {}).get('version')
 48 +         if critpath and advisory and version:
 49 +             self._log('info', "Scheduling openQA jobs for update {0}".format(advisory))
 50 +             # pylint: disable=no-member
 51 +             jobs = schedule.jobs_from_update(advisory, version, openqa_hostname=self.openqa_hostname)
 52 +             if jobs:
 53 +                 self._log('info', "openQA jobs run on update {0}: "
 54 +                           "{1}".format(advisory, ' '.join(str(job) for job in jobs)))
 55 +             else:
 56 +                 self._log('warning', "No openQA jobs run!")
 57 +                 return
 58 + 
 59 +             self._log('debug', "Finished")
 60 +             return
 61 + 
 62 +     def consume(self, message):
 63 +         """Consume incoming message."""
 64 +         if 'pungi' in message['body']['topic']:
 65 +             return self._consume_compose(message)
 66 +         elif 'bodhi' in message['body']['topic']:
 67 +             return self._consume_update(message)
 68 + 
 69   
 70   class OpenQAProductionScheduler(OpenQAScheduler, OpenQAProductionConsumer):
 71       """A scheduling consumer that listens for production fedmsgs and
 72       creates events in the production openQA instance by default.
 73       """
 74 -     topic = ["org.fedoraproject.prod.pungi.compose.status.change"]
 75 +     topic = ["org.fedoraproject.prod.pungi.compose.status.change",
 76 +              "org.fedoraproject.prod.bodhi.update.request.testing",
 77 +              "org.fedoraproject.prod.bodhi.update.edit"]
 78       config_key = "fedora_openqa.scheduler.prod.enabled"
 79   
 80   
 81 @@ -141,7 +170,9 @@
 82       """A scheduling consumer that listens for staging fedmsgs and
 83       creates events in the staging openQA instance by default.
 84       """
 85 -     topic = ["org.fedoraproject.stg.pungi.compose.status.change"]
 86 +     topic = ["org.fedoraproject.stg.pungi.compose.status.change",
 87 +              "org.fedoraproject.stg.bodhi.update.request.testing",
 88 +              "org.fedoraproject.stg.bodhi.update.edit"]
 89       config_key = "fedora_openqa.scheduler.stg.enabled"
 90   
 91   
 92 @@ -149,7 +180,9 @@
 93       """A scheduling consumer that listens for dev fedmsgs and creates
 94       events in a local openQA instance by default.
 95       """
 96 -     topic = ["org.fedoraproject.dev.pungi.compose.status.change"]
 97 +     topic = ["org.fedoraproject.dev.pungi.compose.status.change",
 98 +              "org.fedoraproject.dev.bodhi.update.request.testing",
 99 +              "org.fedoraproject.dev.bodhi.update.edit"]
100       config_key = "fedora_openqa.scheduler.test.enabled"
101       # We just hardcode this here and don't inherit from TestConsumer,
102       # as the config values are intended for the Reporter consumers
  1 @@ -24,15 +24,16 @@
  2   """
  3   
  4   # standard libraries
  5 - import re
  6   import logging
  7 + import re
  8 + from functools import partial
  9   from operator import attrgetter
 10   
 11   # External dependencies
 12   import mwclient.errors
 13   from openqa_client.client import OpenQA_Client
 14   from resultsdb_api import ResultsDBapi, ResultsDBapiException
 15 - from resultsdb_conventions.fedora import FedoraImageResult, FedoraComposeResult
 16 + from resultsdb_conventions.fedora import FedoraImageResult, FedoraComposeResult, FedoraBodhiResult
 17   from wikitcms.wiki import Wiki, ResTuple
 18   
 19   # Internal dependencies
 20 @@ -245,6 +246,11 @@
 21       """
 22       client = OpenQA_Client(openqa_hostname)
 23       jobs = client.get_jobs(jobs=jobs, build=build)
 24 +     # cannot do wiki reporting for update jobs
 25 +     jobs = [job for job in jobs if 'ADVISORY' not in job['settings']]
 26 +     if not jobs:
 27 +         logger.debug("No wiki-reportable jobs: most likely all jobs were update tests")
 28 +         return []
 29       passed_testcases = get_passed_testcases(jobs, client)
 30       logger.info("passed testcases: %s", passed_testcases)
 31   
 32 @@ -311,33 +317,65 @@
 33   
 34       jobs = client.get_jobs(jobs=jobs, build=build)
 35       tcname_safeify = re.compile(r"\W+")
 36 -     target_regex = re.compile(r"^(ISO|HDD)(_\d+)?$")
 37 +     # regex for identifying TEST_TARGET values that suggest an image
 38 +     # specific compose test
 39 +     image_target_regex = re.compile(r"^(ISO|HDD)(_\d+)?$")
 40   
 41       for job in jobs:
 42           # don't report jobs that have clone or user-cancelled jobs, or were obsoleted
 43           if job['clone_id'] is not None or job['result'] == "user_cancelled" or job['result'] == 'obsoleted':
 44               continue
 45   
 46 -         # don't report TEST_TARGET=NONE or jobs that are missing TEST_TARGET
 47 -         if 'TEST_TARGET' not in job['settings']:
 48 -             logger.warning("cannot report job %d because TEST_TARGET variable is missing", job['id'])
 49 -             continue
 50 -         ttarget = job['settings']['TEST_TARGET']
 51 -         if ttarget == "NONE":
 52 -             continue
 53 - 
 54           try:
 55 -             compose = job['settings']['BUILD']
 56 +             build = job['settings']['BUILD']
 57               distri = job['settings']['DISTRI']
 58               version = job['settings']['VERSION']
 59           except KeyError:
 60 -             logger.warning("cannot report job %d because it is missing compose/distri/version", job['id'])
 61 +             logger.warning("cannot report job %d because it is missing build/distri/version", job['id'])
 62               continue
 63   
 64 -         # construct args for resultsdb_conventions Result
 65 -         kwargs = {}
 66 +         # sanitize the test name
 67           tc_name = tcname_safeify.sub("_", job['test']).lower()
 68 -         kwargs["tc_name"] = "compose." + tc_name  # FIXME: this will not be "compose." for update testing...
 69 + 
 70 +         # figure out the resultsdb_convention result type we want and
 71 +         # what the 'item' will be, and create a partial for the Result
 72 +         # class we want to use with the type-specific args
 73 +         ttarget = job['settings'].get('TEST_TARGET', '')
 74 +         rdbpartial = None
 75 +         if 'ADVISORY' in job['settings']:
 76 +             # the 'target' (what will become the 'item' in RDB) is
 77 +             # always the update ID for the update test workflow
 78 +             rdbpartial = partial(FedoraBodhiResult, job['settings']['ADVISORY'], tc_name='update.' + tc_name)
 79 + 
 80 +         elif image_target_regex.match(ttarget):
 81 +             # We have a compose test result for a specific image
 82 +             # 'build' will be the compose ID, job['settings'][ttarget]
 83 +             # will be the filename of the tested image
 84 +             imagename = job['settings'][ttarget]
 85 +             # special case for images decompressed for testing
 86 +             if job['settings']['IMAGETYPE'] == 'raw-xz' and imagename.endswith('.raw'):
 87 +                 imagename += '.xz'
 88 +             rdbpartial = partial(FedoraImageResult, imagename, build, tc_name='compose.' + tc_name)
 89 + 
 90 +         elif ttarget == 'COMPOSE':
 91 +             # We have a non-image-specific compose test result
 92 +             # 'build' will be the compose ID
 93 +             rdbpartial = partial(FedoraComposeResult, build, tc_name='compose.' + tc_name)
 94 + 
 95 +         # don't report TEST_TARGET=NONE, non-update jobs that are
 96 +         # missing TEST_TARGET, or TEST_TARGET values we don't grok
 97 +         if ttarget == "NONE":
 98 +             # this is an explicit 'do not report' setting, so no warn
 99 +             continue
100 +         if not rdbpartial:
101 +             if not ttarget:
102 +                 logger.warning("cannot report job %d because TEST_TARGET variable is missing", job['id'])
103 +             else:
104 +                 logger.warning("Could not understand TEST_TARGET value %s for job %d", ttarget, job['id'])
105 +             continue
106 + 
107 +         # construct common args for resultsdb_conventions Result
108 +         kwargs = {}
109           # map openQA's results to resultsdb's outcome
110           kwargs["outcome"] = {
111               'passed': "PASSED",
112 @@ -353,7 +391,7 @@
113   
114           # create link to overall url for group ref_url
115           overall_url = "%s/tests/overview?distri=%s&version=%s&build=%s" % (
116 -             openqa_baseurl, distri, version, compose)
117 +             openqa_baseurl, distri, version, build)
118   
119           # put in the "note" field whether some module failed
120           for module in job["modules"]:
121 @@ -363,20 +401,13 @@
122               if module["result"] == "failed" and job["result"] == "softfailed":
123                   kwargs["note"] = "non-important module {0} failed".format(module["name"])
124   
125 -         if target_regex.match(ttarget):
126 -             test_target_name = job['settings'][ttarget]
127 -             # special case for images decompressed for testing
128 -             if job['settings']['IMAGETYPE'] == 'raw-xz' and test_target_name.endswith('.raw'):
129 -                 test_target_name += '.xz'
130 -             rdb_object = FedoraImageResult(test_target_name, compose, **kwargs)
131 -         elif ttarget == "COMPOSE":
132 -             rdb_object = FedoraComposeResult(compose, **kwargs)
133 -         else:
134 -             logger.warning("cannot report job %d because TEST_TARGET variable is invalid", job['id'])
135 -             continue
136 +         # create the Result instance
137 +         rdb_object = rdbpartial(**kwargs)
138   
139 +         # Add some more extradata items
140           rdb_object.extradata.update({
141 -             'firmware': 'uefi' if 'UEFI' in job['settings'] else 'bios'
142 +             'firmware': 'uefi' if 'UEFI' in job['settings'] else 'bios',
143 +             'arch': job['settings']['ARCH']
144           })
145           # FIXME: use overall_url as a group ref_url
146   
 1 @@ -261,4 +261,58 @@
 2   
 3       return (rel.cid, jobs)
 4   
 5 + def jobs_from_update(update, version, flavors=None, force=False, extraparams=None, openqa_hostname=None):
 6 +     """Schedule jobs for a specific Fedora update. update is the
 7 +     advisory ID, version is the release number, flavors defines which
 8 +     update tests should be run (valid values are the 'flavdict' keys).
 9 +     force, extraparams and openqa_hostname are as for
10 +     jobs_from_compose.
11 +     """
12 +     version = str(version)
13 +     build = 'Update-{0}'.format(update)
14 +     if extraparams:
15 +         build = '{0}-EXTRA'.format(build)
16 +     flavdict = {
17 +         'server': {
18 +             'HDD_1': 'disk_f{0}_server_3_x86_64.img'.format(version),
19 +         },
20 +         'workstation': {
21 +             'HDD_1': 'disk_f{0}_workstation_3_x86_64.img'.format(version),
22 +             'DESKTOP': 'gnome',
23 +         },
24 +     }
25 +     baseparams = {
26 +         'DISTRI': 'fedora',
27 +         'VERSION': version,
28 +         'ARCH': 'x86_64',
29 +         'BUILD': build,
30 +         'ADVISORY': update,
31 +     }
32 +     client = OpenQA_Client(openqa_hostname)
33 +     jobs = []
34 + 
35 +     if not flavors:
36 +         flavors = flavdict.keys()
37 + 
38 +     for flavor in flavors:
39 +         fullflav = 'updates-{0}'.format(flavor)
40 +         if not force:
41 +             # dupe check
42 +             currjobs = client.openqa_request('GET', 'jobs', params={'build': build})['jobs']
43 +             currjobs = [cjob for cjob in currjobs if cjob['settings']['FLAVOR'] == fullflav]
44 +             if currjobs:
45 +                 logger.info("jobs_from_update: Existing jobs found for update %s flavor %s, and force "
46 +                             "not set! No jobs scheduled.", update, flavor)
47 +                 continue
48 +         flavparams = flavdict[flavor]
49 +         flavparams.update(baseparams)
50 +         flavparams['FLAVOR'] = fullflav
51 +         if extraparams:
52 +             flavparams.update(extraparams)
53 +         output = client.openqa_request('POST', 'isos', flavparams)
54 +         logger.debug("jobs_from_update: planned %s jobs: %s", flavor, output["ids"])
55 +         jobs.extend(output["ids"])
56 + 
57 +     return jobs
58 + 
59   # vim: set textwidth=120 ts=8 et sw=4:
1 @@ -53,7 +53,7 @@
2       url = "https://pagure.io/fedora-qa/fedora_openqa",
3       packages = ["fedora_openqa"],
4       install_requires = ['fedfind>=2.5.0', 'fedmsg', 'openqa-client>=1.1', 'setuptools',
5 -                         'six', 'resultsdb_api', 'resultsdb_conventions>=2.0.0', 'wikitcms'],
6 +                         'six', 'resultsdb_api', 'resultsdb_conventions>=2.0.2', 'wikitcms'],
7       tests_require=['pytest', 'mock'],
8       cmdclass = {'test': PyTest},
9       long_description=read('README.md'),
 1 @@ -36,7 +36,7 @@
 2   
 3   @pytest.fixture(scope="function")
 4   def jobdict01():
 5 -     """An openQA job dict."""
 6 +     """An openQA job dict, for a compose test."""
 7       return {
 8           "children": {
 9               "Chained": [],
10 @@ -116,6 +116,88 @@
11       }
12   
13   @pytest.fixture(scope="function")
14 + def jobdict02():
15 +     """Another openQA job dict, this one for an update test."""
16 +     return {
17 +         "assets": {
18 +             "hdd": ["disk_f25_server_3_x86_64.img"]
19 +         },
20 +         "assigned_worker_id": 8,
21 +         "children": {
22 +             "Chained": [],
23 +             "Parallel": []
24 +         },
25 +         "clone_id": None,
26 +         "group": "fedora",
27 +         "group_id": 1,
28 +         "id": 72517,
29 +         "modules": [
30 +             {
31 +                 "category": "tests",
32 +                 "flags": ["fatal", "milestone"],
33 +                 "name": "_console_wait_login",
34 +                 "result": "passed"
35 +             },
36 +             {
37 +                 "category": "tests",
38 +                 "flags": ["fatal"],
39 +                 "name": "_advisory_update",
40 +                 "result": "passed"
41 +             }
42 +             ,
43 +             {
44 +                 "category": "tests",
45 +                 "flags": ["fatal"],
46 +                 "name": "base_selinux",
47 +                 "result": "passed"
48 +             },
49 +             {
50 +                 "category": "tests",
51 +                 "flags": ["fatal"],
52 +                 "name": "_advisory_post",
53 +                 "result": "passed"
54 +             }
55 +         ],
56 +         "name": "fedora-25-updates-server-x86_64-BuildFEDORA-2017-376ae2b92c-base_selinux@64bit",
57 +         "parents": {
58 +             "Chained": [],
59 +             "Parallel":[]
60 +         },
61 +         "priority": 40,
62 +         "result": "passed",
63 +         "retry_avbl": 3,
64 +         "settings": {
65 +             "ADVISORY": "FEDORA-2017-376ae2b92c",
66 +             "ARCH": "x86_64",
67 +             "BACKEND": "qemu",
68 +             "BOOTFROM": "c",
69 +             "BUILD": "FEDORA-2017-376ae2b92c",
70 +             "CURRREL": "25",
71 +             "DISTRI": "fedora",
72 +             "FLAVOR": "updates-server",
73 +             "HDD_1": "disk_f25_server_3_x86_64.img",
74 +             "MACHINE": "64bit",
75 +             "NAME": "00072517-fedora-25-updates-server-x86_64-BuildFEDORA-2017-376ae2b92c-base_selinux@64bit",
76 +             "PART_TABLE_TYPE": "mbr",
77 +             "POSTINSTALL": "base_selinux",
78 +             "PREVREL": "24",
79 +             "QEMUCPU": "Nehalem",
80 +             "QEMUCPUS": "2",
81 +             "QEMURAM": "2048",
82 +             "QEMUVGA": "qxl",
83 +             "ROOT_PASSWORD": "weakpassword",
84 +             "START_AFTER_TEST": "install_default_upload",
85 +             "TEST": "base_selinux",
86 +             "USER_LOGIN": "false",
87 +             "VERSION": "25"
88 +         },
89 +         "state": "done",
90 +         "t_finished": "2017-02-22T23:13:13",
91 +         "t_started": "2017-02-22T23:07:29",
92 +         "test": "base_selinux"
93 +     }
94 + 
95 + @pytest.fixture(scope="function")
96   def ffimg01():
97       """A pre-canned fedfind image dict, for the x86_64 Server DVD from
98       the 20170207.n.0 Rawhide nightly.
 1 @@ -92,6 +92,51 @@
 2           # should exit 1
 3           assert excinfo.value.code == 1
 4   
 5 + @mock.patch('fedora_openqa.schedule.jobs_from_update', return_value=[1, 2], autospec=True)
 6 + def test_command_update(fakejfu, capsys):
 7 +     """Test the command_update function."""
 8 +     args = cli.parse_args(
 9 +         ['update', 'FEDORA-2017-b07d628952', '25']
10 +     )
11 +     with pytest.raises(SystemExit) as excinfo:
12 +         cli.command_update(args)
13 +     (out, _) = capsys.readouterr()
14 +     # should print out list of scheduled jobs
15 +     assert out == "Scheduled jobs: 1, 2\n"
16 +     # should exit 0
17 +     assert not excinfo.value.code
18 +     # shouldn't force
19 +     assert fakejfu.call_args[1]['force'] is False
20 + 
21 +     # check 'flavor'
22 +     args = cli.parse_args(
23 +         ['update', 'FEDORA-2017-b07d628952', '25', '--flavor', 'server']
24 +     )
25 +     with pytest.raises(SystemExit) as excinfo:
26 +         cli.command_update(args)
27 +     # should exit 0
28 +     assert not excinfo.value.code
29 +     assert fakejfu.call_args[1]['flavors'] == ['server']
30 + 
31 +     # check 'force'
32 +     args = cli.parse_args(
33 +         ['update', 'FEDORA-2017-b07d628952', '25', '--force']
34 +     )
35 +     with pytest.raises(SystemExit) as excinfo:
36 +         cli.command_update(args)
37 +     # should exit 0
38 +     assert not excinfo.value.code
39 +     assert fakejfu.call_args[1]['force'] is True
40 + 
41 +     # check 'openqa_hostname'
42 +     args = cli.parse_args(
43 +         ['update', 'FEDORA-2017-b07d628952', '25', '--openqa-hostname', 'openqa.example']
44 +     )
45 +     with pytest.raises(SystemExit) as excinfo:
46 +         cli.command_update(args)
47 +     # should exit 0
48 +     assert not excinfo.value.code
49 +     assert fakejfu.call_args[1]['openqa_hostname'] == 'openqa.example'
50   
51   @pytest.mark.parametrize(
52       "jobargs,argname,expecteds",
  1 @@ -25,6 +25,9 @@
  2   from __future__ import unicode_literals
  3   from __future__ import print_function
  4   
  5 + # stdlib imports
  6 + import copy
  7 + 
  8   # external imports
  9   import mock
 10   import pytest
 11 @@ -124,6 +127,70 @@
 12       }
 13   }
 14   
 15 + # Critpath update creation message. These are huge, so this is heavily
 16 + # edited.
 17 + CRITPATHCREATE = {
 18 +     "body": {
 19 +         "i": 1,
 20 +         "msg": {
 21 +             "agent": "msekleta",
 22 +             "update": {
 23 +                 "alias": "FEDORA-2017-ea07abb5d5",
 24 +                 "critpath": True,
 25 +                 "release": {
 26 +                     "branch": "f24",
 27 +                     "dist_tag": "f24",
 28 +                     "id_prefix": "FEDORA",
 29 +                     "long_name": "Fedora 24",
 30 +                     "name": "F24",
 31 +                     "version": "24"
 32 +                 },
 33 +             },
 34 +         },
 35 +         "msg_id": "2017-a6e10ab5-f861-4671-8945-ac1cf004a474",
 36 +         "source_name": "datanommer",
 37 +         "source_version": "0.6.5",
 38 +         "timestamp": 1487862992.0,
 39 +         "topic": "org.fedoraproject.prod.bodhi.update.request.testing"
 40 +     }
 41 + }
 42 + 
 43 + # Non-critpath update creation message
 44 + NONCRITCREATE = copy.deepcopy(CRITPATHCREATE)
 45 + NONCRITCREATE['body']['msg']['update']['critpath'] = False
 46 + 
 47 + # Critpath update edit message
 48 + CRITPATHEDIT = {
 49 +     "body": {
 50 +         "i": 1,
 51 +         "msg": {
 52 +             "agent": "hobbes1069",
 53 +             "update": {
 54 +                 "alias": "FEDORA-2017-e6d7184200",
 55 +                 "critpath": True,
 56 +                 "release": {
 57 +                     "branch": "f24",
 58 +                     "dist_tag": "f24",
 59 +                     "id_prefix": "FEDORA",
 60 +                     "long_name": "Fedora 24",
 61 +                     "name": "F24",
 62 +                     "version": "24"
 63 +                 },
 64 +             },
 65 +         },
 66 +         "msg_id": "2017-7213730f-40c8-4e27-9135-630f5de2113d",
 67 +         "source_name": "datanommer",
 68 +         "source_version": "0.6.5",
 69 +         "timestamp": 1487735650.0,
 70 +         "topic": "org.fedoraproject.prod.bodhi.update.edit"
 71 +     }
 72 + }
 73 + 
 74 + # Non-critpath update edit message
 75 + NONCRITEDIT = copy.deepcopy(CRITPATHEDIT)
 76 + NONCRITEDIT['body']['msg']['update']['critpath'] = False
 77 + 
 78 + 
 79   # proper consumer init requires a fedmsg hub instance, we don't have
 80   # one and don't want to faff around faking one.
 81   with mock.patch('fedmsg.consumers.FedmsgConsumer.__init__', return_value=None):
 82 @@ -151,6 +218,7 @@
 83       """Tests for the consumers."""
 84   
 85       @mock.patch('fedora_openqa.schedule.jobs_from_compose', return_value=('somecompose', [1]), autospec=True)
 86 +     @mock.patch('fedora_openqa.schedule.jobs_from_update', return_value=[1], autospec=True)
 87       @pytest.mark.parametrize(
 88           "consumer,oqah",
 89           [
 90 @@ -166,9 +234,13 @@
 91               (DOOMEDCOMPOSE, False),
 92               (FINISHEDCOMPOSE, True),
 93               (FINCOMPLETE, True),
 94 +             (CRITPATHCREATE, True),
 95 +             (CRITPATHEDIT, True),
 96 +             (NONCRITCREATE, False),
 97 +             (NONCRITEDIT, False),
 98           ]
 99       )
100 -     def test_scheduler(self, fake_schedule, consumer, oqah, message, create):
101 +     def test_scheduler(self, fake_update, fake_schedule, consumer, oqah, message, create):
102           """Test the job scheduling consumers do their thing. The
103           parametrization pairs are:
104           1. (consumer, expected openQA hostname)
105 @@ -179,11 +251,14 @@
106           """
107           consumer.consume(message)
108           if create:
109 -             assert fake_schedule.call_count == 1
110 -             assert fake_schedule.call_args[1]['openqa_hostname'] == oqah
111 +             assert fake_schedule.call_count + fake_update.call_count == 1
112 +             if fake_schedule.call_count == 1:
113 +                 assert fake_schedule.call_args[1]['openqa_hostname'] == oqah
114 +             else:
115 +                 assert fake_update.call_args[1]['openqa_hostname'] == oqah
116           else:
117 -             assert fake_schedule.call_count == 0
118 -         fake_schedule.reset_mock()
119 +             assert fake_schedule.call_count + fake_update.call_count == 0
120 +         #fake_schedule.reset_mock()
121   
122   
123       @mock.patch('fedora_openqa.report.wiki_report', autospec=True)
 1 @@ -334,6 +334,18 @@
 2           # check results didn't happen
 3           assert mockinst.report_validation_results.call_args is None
 4   
 5 +     def  test_update_noreport(self, fake_getpassed, wikimock, oqaclientmock, jobdict02):
 6 +         """Check we do no reporting (but don't crash or do anything
 7 +         else odd) for an update test job.
 8 +         """
 9 +         # adjust the OpenQA_Client instance mock to return jobdict02
10 +         instmock = oqaclientmock[1]
11 +         instmock.get_jobs.return_value = [jobdict02]
12 +         (_, mockinst) = wikimock
13 +         ret = fosreport.wiki_report(jobs=[1])
14 +         # check results didn't happen
15 +         assert mockinst.report_validation_results.call_args is None
16 +         assert ret == []
17   
18   @mock.patch.object(resultsdb_api.ResultsDBapi, 'create_result')
19   @pytest.mark.usefixtures("ffmock", "oqaclientmock")
20 @@ -349,6 +361,20 @@
21           assert fakeres.call_args[1]['firmware'] == 'bios'
22           assert fakeres.call_args[1]['outcome'] == 'PASSED'
23   
24 +     def test_update(self, fakeres, oqaclientmock, jobdict02):
25 +         """Check report behaviour with an update test job (rather than
26 +         a compose test job).
27 +         """
28 +         # adjust the OpenQA_Client instance mock to return jobdict02
29 +         instmock = oqaclientmock[1]
30 +         instmock.get_jobs.return_value = [jobdict02]
31 +         fosreport.resultsdb_report(jobs=[1])
32 +         assert fakeres.call_args[1]['item'] == 'FEDORA-2017-376ae2b92c'
33 +         assert fakeres.call_args[1]['ref_url'] == 'https://some.url/tests/72517'
34 +         assert fakeres.call_args[1]['testcase']['name'] == 'update.base_selinux'
35 +         assert fakeres.call_args[1]['firmware'] == 'bios'
36 +         assert fakeres.call_args[1]['outcome'] == 'PASSED'
37 + 
38       def test_uefi(self, fakeres, oqaclientmock):
39           """Check resultsdb_report with UEFI test."""
40           # modify the job dict used by the mock fixture
  1 @@ -364,4 +364,99 @@
  2           with pytest.raises(schedule.TriggerException):
  3               ret = schedule.jobs_from_compose(COMPURL)
  4   
  5 + @mock.patch('fedora_openqa.schedule.OpenQA_Client', autospec=True)
  6 + def test_jobs_from_update(fakeclient):
  7 +     """Tests for jobs_from_update."""
  8 +     # the OpenQA_Client instance mock
  9 +     fakeinst = fakeclient.return_value
 10 +     # for now, return no 'jobs' (for the dupe query), one 'id' (for
 11 +     # the post request)
 12 +     fakeinst.openqa_request.return_value = {'jobs': [], 'ids': [1]}
 13 +     # simple case
 14 +     ret = schedule.jobs_from_update('FEDORA-2017-b07d628952', '25')
 15 +     # should get two jobs (as we schedule for two flavors by default)
 16 +     assert ret == [1, 1]
 17 +     # find the POST calls
 18 +     posts = [call for call in fakeinst.openqa_request.call_args_list if call[0][0] == 'POST']
 19 +     # two flavors by default, two calls
 20 +     assert len(posts) == 2
 21 +     parmdicts = [call[0][2] for call in posts]
 22 +     parmdicts.sort()
 23 +     assert parmdicts == [
 24 +         {
 25 +             'DISTRI': 'fedora',
 26 +             'VERSION': '25',
 27 +             'ARCH': 'x86_64',
 28 +             'BUILD': 'Update-FEDORA-2017-b07d628952',
 29 +             'ADVISORY': 'FEDORA-2017-b07d628952',
 30 +             'HDD_1': 'disk_f25_server_3_x86_64.img',
 31 +             'FLAVOR': 'updates-server',
 32 +         },
 33 +         {
 34 +             'DISTRI': 'fedora',
 35 +             'VERSION': '25',
 36 +             'ARCH': 'x86_64',
 37 +             'BUILD': 'Update-FEDORA-2017-b07d628952',
 38 +             'ADVISORY': 'FEDORA-2017-b07d628952',
 39 +             'HDD_1': 'disk_f25_workstation_3_x86_64.img',
 40 +             'FLAVOR': 'updates-workstation',
 41 +             'DESKTOP': 'gnome',
 42 +         }
 43 +     ]
 44 + 
 45 +     # test 'flavors'
 46 +     fakeinst.openqa_request.reset_mock()
 47 +     ret = schedule.jobs_from_update('FEDORA-2017-b07d628952', '25', flavors=['server'])
 48 +     # should get one job
 49 +     assert ret == [1]
 50 +     # find the POST calls
 51 +     posts = [call for call in fakeinst.openqa_request.call_args_list if call[0][0] == 'POST']
 52 +     # one flavor, one call
 53 +     assert len(posts) == 1
 54 +     # check parm dict FLAVOR value
 55 +     assert posts[0][0][2]['FLAVOR'] == 'updates-server'
 56 + 
 57 +     # test dupe detection and 'force'
 58 +     fakeinst.openqa_request.reset_mock()
 59 +     # this looks like a 'dupe' for the server flavor
 60 +     fakeinst.openqa_request.return_value = {
 61 +         'jobs': [
 62 +             {
 63 +                 'settings': {
 64 +                     'FLAVOR': 'updates-server',
 65 +                 },
 66 +             },
 67 +         ],
 68 +         'ids': [1],
 69 +     }
 70 +     ret = schedule.jobs_from_update('FEDORA-2017-b07d628952', '25')
 71 +     # should get one job, as we shouldn't POST for server
 72 +     assert ret == [1]
 73 +     # find the POST calls
 74 +     posts = [call for call in fakeinst.openqa_request.call_args_list if call[0][0] == 'POST']
 75 +     # one flavor, one call
 76 +     assert len(posts) == 1
 77 +     # check parm dict FLAVOR value
 78 +     assert posts[0][0][2]['FLAVOR'] == 'updates-workstation'
 79 +     # now try with force=True
 80 +     fakeinst.openqa_request.reset_mock()
 81 +     ret = schedule.jobs_from_update('FEDORA-2017-b07d628952', '25', force=True)
 82 +     # should get two jobs this time
 83 +     assert ret == [1, 1]
 84 + 
 85 +     # test extraparams
 86 +     fakeinst.openqa_request.reset_mock()
 87 +     # set the openqa_request return value back to the no-dupes version
 88 +     fakeinst.openqa_request.return_value = {'jobs': [], 'ids': [1]}
 89 +     ret = schedule.jobs_from_update('FEDORA-2017-b07d628952', '25', flavors=['server'], extraparams={'FOO': 'bar'})
 90 +     # find the POST calls
 91 +     posts = [call for call in fakeinst.openqa_request.call_args_list if call[0][0] == 'POST']
 92 +     # check parm dict values
 93 +     assert posts[0][0][2]['BUILD'] == 'Update-FEDORA-2017-b07d628952-EXTRA'
 94 +     assert posts[0][0][2]['FOO'] == 'bar'
 95 + 
 96 +     # test openqa_hostname
 97 +     ret = schedule.jobs_from_update('FEDORA-2017-b07d628952', '25', openqa_hostname='openqa.example')
 98 +     assert fakeclient.call_args[0][0] == 'openqa.example'
 99 + 
100   # vim: set textwidth=120 ts=8 et sw=4: