Commit 348ae48 Implement a 'whitelist' for running non-critpath update tests

3 files Authored and Committed by adamwill a year ago
Implement a 'whitelist' for running non-critpath update tests

Summary:
We want to be able to selectively run the update tests for
non-critpath updates. This implements a system for doing so:
a whitelist where we can indicate that some or all of the
update tests should be run for specific package names. For any
given package we can specify an iterable of flavors to run the
tests for, or just indicate "run all the update tests".

The whitelist is implemented much like the WANTED list: there's
a default whitelist in `config.py` which can be overridden if
desired by a JSON-formatted config file.

The whitelist is parsed by the fedmsg consumer that automatically
schedules update jobs. Of course, we can still manually trigger
the tests for *any* update we like via the CLI.

Test Plan:
The new feature should be covered pretty well in the
test suite. Otherwise, you have to just deploy this on an instance
with the fedmsg listening stuff set up, and wait for or replay
a message for an update that's on the whitelist. I am going to
deploy this to staging so we can check it works properly.

Reviewers: jsedlak, jskladan

Reviewed By: jsedlak

Subscribers: tflink

Differential Revision: https://phab.qa.fedoraproject.org/D1190

    
 1 @@ -222,9 +222,31 @@
 2       },
 3   ]
 4   
 5 + # Whitelist of non-critpath package names to run update tests on.
 6 + # Dict keys are the package names, value is either an iterable of the
 7 + # flavor(s) of update tests to run for that package or can just be
 8 + # None (or anything else false-y) which means "run all the flavors".
 9 + UPDATEWL = {
10 +     # FreeIPA-related bits
11 +     '389-ds': ('server',),
12 +     '389-ds-base': ('server',),
13 +     'bind': ('server',),
14 +     'bind-dyndb-ldap': ('server',),
15 +     'certmonger': ('server',),
16 +     'ding-libs': ('server',),
17 +     'freeipa': ('server',),
18 +     'krb5-server': ('server',),
19 +     'pki-core': ('server',),
20 +     'sssd': ('server',),
21 +     'tomcat': ('server',),
22 +     # PostgreSQL is a release-blocking server role
23 +     'postgresql': ('server',),
24 + }
25 + 
26   for path in ('/etc/fedora-openqa',
27                '{0}/.config/fedora-openqa'.format(os.path.expanduser('~'))):
28       try:
29 +         # load WANTED override config file
30           fname = '{0}/images.json'.format(path)
31           with open(fname, 'r') as fout:
32               WANTED = json.load(fout)
33 @@ -237,4 +259,13 @@
34           # file not found
35           pass
36   
37 +     try:
38 +         # load UPDATEWL override config file
39 +         fname = '{0}/updatewl.json'.format(path)
40 +         with open(fname, 'r') as fout:
41 +             UPDATEWL = json.load(fout)
42 +     except IOError:
43 +         # file not found
44 +         pass
45 + 
46   # vim: set textwidth=120 ts=8 et sw=4:
 1 @@ -24,7 +24,7 @@
 2   import fedmsg.consumers
 3   
 4   # internal imports
 5 - from .config import CONFIG
 6 + from .config import CONFIG, UPDATEWL
 7   from . import schedule
 8   from . import report
 9   
10 @@ -136,20 +136,59 @@
11           version = update.get('release', {}).get('version')
12           # to make sure this is a Fedora, not EPEL, update
13           idpref = update.get('release', {}).get('id_prefix')
14 +         # list of flavors to run the tests for, starts empty. If set
15 +         # to None, means 'run all tests'.
16 +         flavors = []
17 + 
18 +         # if update is critpath, always run all update tests
19           if critpath and advisory and version and idpref == 'FEDORA':
20 -             self._log('info', "Scheduling openQA jobs for update {0}".format(advisory))
21 -             # pylint: disable=no-member
22 -             jobs = schedule.jobs_from_update(advisory, version, openqa_hostname=self.openqa_hostname, force=True)
23 -             if jobs:
24 -                 self._log('info', "openQA jobs run on update {0}: "
25 -                           "{1}".format(advisory, ' '.join(str(job) for job in jobs)))
26 -             else:
27 -                 self._log('warning', "No openQA jobs run!")
28 -                 return
29 +             self._log('info', "Scheduling openQA jobs for critical path update {0}".format(advisory))
30 +             flavors = None
31 + 
32 +         # otherwise check the whitelist
33 +         elif advisory and version and idpref == 'FEDORA':
34 +             self._log('debug', "Checking whitelist for update {0}".format(advisory))
35 +             for build in update.get('builds', []):
36 +                 # get just the package name by splitting the NVR. This
37 +                 # assumes all NVRs actually contain a V and an R.
38 +                 # Happily, RPM rejects dashes in version or release.
39 +                 pkgname = build['nvr'].rsplit('-', 2)[0]
40 +                 # now check the whitelist and adjust flavors
41 +                 if pkgname in UPDATEWL:
42 +                     if not UPDATEWL[pkgname]:
43 +                         # this means *all* flavors, and we can short
44 +                         self._log('info', "Running ALL openQA tests for whitelisted update {0}".format(advisory))
45 +                         flavors = None
46 +                         break
47 +                     else:
48 +                         flavors.extend(UPDATEWL[pkgname])
49 + 
50 +         if flavors:
51 +             # this means we have a list of flavors, not None indicating
52 +             # *all* flavors, let's log that
53 +             tmpl = "Running update tests for flavors {0} for whitelisted update {1}"
54 +             self._log('info', tmpl.format(', '.join(flavors), advisory))
55 +         elif flavors == []:
56 +             # This means we ain't running nothin'
57 +             self._log('debug', "Update is not critical path and no packages in whitelist, no jobs scheduled")
58 +             return
59   
60 -             self._log('debug', "Finished")
61 +         # Finally, now we've decided on flavors, run the jobs. flavors
62 +         # being None here results in our desired behaviour (jobs will
63 +         # be created for *all* flavors)
64 +         # pylint: disable=no-member
65 +         jobs = schedule.jobs_from_update(
66 +             advisory, version, flavors=flavors, openqa_hostname=self.openqa_hostname, force=True)
67 +         if jobs:
68 +             self._log('info', "openQA jobs run on update {0}: "
69 +                       "{1}".format(advisory, ' '.join(str(job) for job in jobs)))
70 +         else:
71 +             self._log('warning', "No openQA jobs run!")
72               return
73   
74 +         self._log('debug', "Finished")
75 +         return
76 + 
77       def consume(self, message):
78           """Consume incoming message."""
79           if 'pungi' in message['body']['topic']:
  1 @@ -34,6 +34,15 @@
  2   
  3   # 'internal' imports
  4   import fedora_openqa.consumer
  5 + from fedora_openqa.config import UPDATEWL
  6 + 
  7 + # Modified version of the default UPDATEWL for testing purposes
  8 + # we have to do this because we don't have any real-world cases
  9 + # for whitelisting anything but the 'server' tests yet, so we can't
 10 + # fully test the feature with the real default whitelist
 11 + MODIFIEDWL = copy.deepcopy(UPDATEWL)
 12 + MODIFIEDWL['gnome-terminal'] = ('workstation',)
 13 + MODIFIEDWL['kernel'] = None
 14   
 15   # Passed test message
 16   PASSMSG = {
 17 @@ -136,6 +145,13 @@
 18               "agent": "msekleta",
 19               "update": {
 20                   "alias": "FEDORA-2017-ea07abb5d5",
 21 +                 "builds": [
 22 +                     {
 23 +                         "epoch": 0,
 24 +                         "nvr": "systemd-231-14.fc24",
 25 +                         "signed": False
 26 +                     }
 27 +                 ],
 28                   "critpath": True,
 29                   "release": {
 30                       "branch": "f24",
 31 @@ -155,10 +171,15 @@
 32       }
 33   }
 34   
 35 - # Non-critpath update creation message
 36 + # Non-critpath, non-whitelisted update creation message
 37   NONCRITCREATE = copy.deepcopy(CRITPATHCREATE)
 38   NONCRITCREATE['body']['msg']['update']['critpath'] = False
 39   
 40 + # Non-critpath, one-flavor-whitelisted update creation message
 41 + WLCREATE = copy.deepcopy(CRITPATHCREATE)
 42 + WLCREATE['body']['msg']['update']['critpath'] = False
 43 + WLCREATE['body']['msg']['update']['builds'] = [{"epoch": 0, "nvr": "freeipa-4.4.4-1.fc24", "signed": False}]
 44 + 
 45   # Critpath EPEL update creation message
 46   EPELCREATE = copy.deepcopy(CRITPATHCREATE)
 47   EPELCREATE['body']['msg']['update']['release']['id_prefix'] = 'FEDORA-EPEL'
 48 @@ -190,10 +211,23 @@
 49       }
 50   }
 51   
 52 - # Non-critpath update edit message
 53 + # Non-critpath, non-whitelisted update edit message
 54   NONCRITEDIT = copy.deepcopy(CRITPATHEDIT)
 55   NONCRITEDIT['body']['msg']['update']['critpath'] = False
 56   
 57 + # Non-critpath, two-flavors-whitelisted update edit message
 58 + WLEDIT = copy.deepcopy(CRITPATHEDIT)
 59 + WLEDIT['body']['msg']['update']['critpath'] = False
 60 + WLEDIT['body']['msg']['update']['builds'] = [
 61 +     {"epoch": 0, "nvr": "freeipa-4.4.4-1.fc26", "signed": False},
 62 +     {"epoch": 0, "nvr": "gnome-terminal-3.24.1-1.fc24", "signed": False},
 63 + ]
 64 + 
 65 + # Non-critpath, all-flavors-whitelisted update edit message
 66 + WLALLEDIT = copy.deepcopy(CRITPATHEDIT)
 67 + WLALLEDIT['body']['msg']['update']['critpath'] = False
 68 + WLALLEDIT['body']['msg']['update']['builds'] = [{"epoch": 0, "nvr": "kernel-4.10.12-100.fc24", "signed": False}]
 69 + 
 70   # Critpath EPEL update edit message
 71   EPELEDIT = copy.deepcopy(CRITPATHEDIT)
 72   EPELEDIT['body']['msg']['update']['release']['id_prefix'] = 'FEDORA-EPEL'
 73 @@ -225,6 +259,7 @@
 74   class TestConsumers:
 75       """Tests for the consumers."""
 76   
 77 +     @mock.patch('fedora_openqa.consumer.UPDATEWL', MODIFIEDWL)
 78       @mock.patch('fedora_openqa.schedule.jobs_from_compose', return_value=('somecompose', [1]), autospec=True)
 79       @mock.patch('fedora_openqa.schedule.jobs_from_update', return_value=[1], autospec=True)
 80       @pytest.mark.parametrize(
 81 @@ -236,38 +271,53 @@
 82           ]
 83       )
 84       @pytest.mark.parametrize(
 85 -         "message,create",
 86 +         "message,flavors",
 87           [
 88 +             # For 'flavors': 'False' means no jobs should be created. For
 89 +             # compose scheduling, any other value just means "jobs should
 90 +             # be created". For update scheduling, any value but 'False'
 91 +             # means "jobs should be created, and this is the expected value
 92 +             # of the 'flavors' kwarg to the fake_update call" (remember,
 93 +             # None means "run tests for all flavors").
 94               (STARTEDCOMPOSE, False),
 95               (DOOMEDCOMPOSE, False),
 96               (FINISHEDCOMPOSE, True),
 97               (FINCOMPLETE, True),
 98 -             (CRITPATHCREATE, True),
 99 -             (CRITPATHEDIT, True),
100 +             # for all critpath updates we should schedule for all flavors
101 +             (CRITPATHCREATE, None),
102 +             (CRITPATHEDIT, None),
103               (NONCRITCREATE, False),
104               (NONCRITEDIT, False),
105               (EPELCREATE, False),
106               (EPELEDIT, False),
107 +             # WLCREATE contains only a 'server'-whitelisted package
108 +             (WLCREATE, ['server']),
109 +             # WLEDIT contains both 'server' and 'workstation'-whitelisted
110 +             # packages
111 +             (WLEDIT, ['server', 'workstation']),
112 +             # WLALLEDIT contains an 'all flavors'-whitelisted package
113 +             (WLALLEDIT, None),
114           ]
115       )
116 -     def test_scheduler(self, fake_update, fake_schedule, consumer, oqah, message, create):
117 +     def test_scheduler(self, fake_update, fake_schedule, consumer, oqah, message, flavors):
118           """Test the job scheduling consumers do their thing. The
119           parametrization pairs are:
120           1. (consumer, expected openQA hostname)
121 -         2. (fedmsg, should job scheduling happen?)
122 +         2. (fedmsg, job creation? / expected flavors value (see above)
123           If jobs are expected, we check the schedule function was hit
124           and that the hostname is as expected. If jobs aren't expected,
125           we check the schedule function was not hit.
126           """
127           consumer.consume(message)
128 -         if create:
129 +         if flavors is False:
130 +             assert fake_schedule.call_count + fake_update.call_count == 0
131 +         else:
132               assert fake_schedule.call_count + fake_update.call_count == 1
133               if fake_schedule.call_count == 1:
134                   assert fake_schedule.call_args[1]['openqa_hostname'] == oqah
135               else:
136                   assert fake_update.call_args[1]['openqa_hostname'] == oqah
137 -         else:
138 -             assert fake_schedule.call_count + fake_update.call_count == 0
139 +                 assert fake_update.call_args[1]['flavors'] == flavors
140           #fake_schedule.reset_mock()
141   
142