From 569ff2c8f26207c08e2c28d1e06bc823932f63d2 Mon Sep 17 00:00:00 2001 From: Lukas Ruzicka Date: Jul 19 2019 12:17:15 +0000 Subject: Test all modules. Fix arguments. Fix arguments. Clean installation. Some fix. Improve logging and ux. Improve logging and ux. Enhance features. Fix typo. Fix typo. Fix name. Fix prec. Fix prec. Fix log. Fix listing profiles. Fix cache cleaning. Fix cache cleaning. Add sleep. Add sleep. Add sleep. Add sleep. Add sleep. Remove sleep and add arguments. Fix installroot. Fix installroot. Fix variables. Fix variables. Fix variables. Add comments to explain the code. Add some logging. --- diff --git a/modular_functions.py b/modular_functions.py index 803d973..5f1a390 100755 --- a/modular_functions.py +++ b/modular_functions.py @@ -25,7 +25,7 @@ import subprocess import sys class DNFoutput: - # This class gathers methods from DNF that provide the DNF output in form of a json for better parsing. + # This class uses the DNF code to communicate with DNF in order to get a correct and parsed list of all modules to further processing. def __init__(self): self.base = dnf.Base() self.base.read_all_repos() @@ -109,6 +109,11 @@ class TestSuite: logging.info('The test suite has been initialized.') self.releasever = None self.fake = False + self.clean_installation() + #self.install = "--installroot=/testinstall" + #self.relver = f"--releasever={self.releasever}" + #self.extras1 = f"--setopt=module_platform_id=platform:f{self.releasever}" + #self.extras2 = f"--setopt=tsflags=justdb" def store_results(self, key, value): if key in self.results.keys(): @@ -122,17 +127,18 @@ class TestSuite: def module_list(self): - + # Get the modular data from the DNF. It comes as a list of dictionaries. processed = self.dnf.output() - # Let us parse the DNF command output and create module dictionaries with all required info, that is name, streams, profiles + # Let us parse the data and for each module create a dictionary, where name is the key and the rest comes as a value. modulelist = {} for line in processed: module = {} name = line['name'] module['name'] = name stream = line['stream'] - # Parsing stream to find default stream if provided + # If the stream is a default stream, we can find a '[d]' symbol in it. + # We record the default stream elsewhere and delete the symbol. if '[d]' in stream: stream = stream.split('[')[0].strip() module['default-stream'] = stream @@ -140,30 +146,26 @@ class TestSuite: stream = stream.split('[')[0].strip() module['stream'] = stream - # Parsing profiles to find default profile, if available + # There might be more profiles and one can be set as a default profile. + # This procedure breaks the list of profiles and finds a default one if present. profiles = line['profiles'].split(',') nprofiles = [] for profile in profiles: - # The main regex also returns profiles for modules with no profiles defined, because - # it eats the first word from the info. We can track such cases down, when the info starts - # with a capital letter. Otherwise, there is a risk of bad parsing, but those cases should be - # quite limited. - n = re.match(r"[A-Z]\w+", profile) - if n: - profile = 'none' if '[d]' in profile: profile = profile.split('[')[0].strip() module['default-profile'] = profile nprofiles.append(profile) + elif '[i]' in profile: + profile = profile.split('[')[0].strip() + nprofiles.append(profile) else: nprofiles.append(profile) module['profiles'] = nprofiles - - info = line['summary'] module['info'] = info - # Recording the module into a module database to be used by other methods. + # Now, let us make a database (dict) with all the modules, with name as keys and list of dicts + # for various streams as values. if name not in modulelist.keys(): #If this is first occurence modulelist[name] = [module] else: # If this is another occurence of same module with different streams. @@ -173,8 +175,8 @@ class TestSuite: pass self.modlist = modulelist - logging.info('The "dnf module list" operation was succesful.') - # Attempt to use dnf outputter. + + logging.debug('The "dnf module list" operation was succesful. Data have been obtained from the DNF.') return self.modlist def module_info(self, moduleName): @@ -183,11 +185,15 @@ class TestSuite: module = modules[moduleName] info = {} for s in module: - info[s['stream']] = s['profile'] + info[s['stream']] = s['profiles'] + logging.debug("The module_info method ran successfully.") return info def use_module(self, module, operation, stream=None, profile=None): + """This method calls different modular commands.""" + # Each time, we need to call a modular dnf command, we will call this method + # with different arguments. if stream == None: smodifier = '' else: @@ -198,32 +204,66 @@ class TestSuite: pmodifier = f"/{profile}" modifier = f"{module}{smodifier}{pmodifier}" key = f"dnf module {operation} {modifier}" + # For some tests, it is good, when we use a fake environment via installroot and we limit + # the DNF operations to "justdb" (only make changes in the database without installing anything). + # This is very useful in case of multiple operations that are much faster. To use this fake option + # do not forget to set_fake(True) before the process. if self.fake == True: releasever = self.releasever - install = f"--installroot=/testinstall" - relver = f"--releasever={releasever}" - extras1 = f"--setopt=module_platform_id=platform:f{releasever}" - extras2 = f"--setopt=tsflags=justdb" - raw = subprocess.run(['dnf', 'module', operation, '-y', modifier, extras], capture_output=True) + install = self.install + relver = self.relver + extras1 = self.extras1 + extras2 = self.extras2 + raw = subprocess.run(['dnf', 'module', operation, '-y', modifier, install, relver, extras1, extras2], capture_output=True) + logging.debug(raw.stdout.decode('utf-8')) else: + # When the command should be real, we will use a different command. This will make changes to the host system. raw = subprocess.run(['dnf', 'module', operation, '-y', modifier], capture_output=True) self.outputs[operation] = raw.stdout.decode('utf-8') if raw.returncode == 0: - logging.info(f"The operation {key} has finished successfully.") result = raw.returncode + logging.info(f"The {key} operation ran successfully.") + result = 0 else: problem = raw.stderr.decode('utf-8') - logging.warning(f"The operation {key} has NOT finished successfully, because of {problem}.") + logging.error(f"The operation {key} has NOT finished successfully, because of {problem}.") print('There was a DNF problem! Check the log for further information.') result = 1 self.store_results(f'dnf module {operation}', result) return result + def clean_installation(self): + """Cleans installation data in fake environment.""" + # All fake tests run in installroot=/testinstall, before each test, the installroot + # should be deleted to achieve a clean test environment. + raw = subprocess.run(['rm', '-rf', '/testinstall'], capture_output=True) + if raw.returncode == 0: + print("\n======== The test environment has been cleaned. ========\n") + logging.info("The test environment has been wiped succesfully.") + else: + print("\n======== There was a problem deleting the test environment, because it had been already deleted or never existed. ========\n") + raw = raw.stderr.decode('utf-8') + logging.error(f"The test date have not been wiped succesfully: {raw} ") + + def is_listed(self, module, stream, profile, inlist): + """Returns if the specified module is listed in dnf module list.""" option = f"--{inlist}" key = f"dnf module list {option}" - raw = subprocess.run(['dnf', 'module', 'list', option], capture_output=True) + # If we operate in the fake environment, we must list in that environment, too, otherwise + # we will never get the correct information because the situation is different on the + # host computer. + if self.fake == True: + releasever = self.releasever + install = self.install + relver = self.relver + extras1 = self.extras1 + extras2 = self.extras2 + raw = subprocess.run(['dnf', 'module', 'list', option, install, relver, extras1, extras2], capture_output=True) + else: + raw = subprocess.run(['dnf', 'module', 'list', option], capture_output=True) + # If the list command ran successfully, parse the list and provide result. if raw.returncode == 0: raw = raw.stdout.decode('utf-8') self.outputs[option] = raw @@ -245,16 +285,24 @@ class TestSuite: result = modfound + strfound + profound if result == 0: break + # The result show a 1 for every error. Therefore, it can be easily spotted, where the problem is + # even if you do now read the log file. + # 0 - everything OK + # 1 - the specified profile is not in list, but the name and stream are + # 10 - the specified stream is not in list, but the name and profile are + # 11 - the specified stream and profile are not in list, but the name is + # 111 - nothing is listed result = modfound + strfound + profound - logging.info(f"The operation {key} has finished successfully.") + logging.info(f"The {key} command ran successfully.") else: problem = raw.stderr.decode('utf-8') - logging.warning(f"The operation {key} has NOT finished successfully, because of {problem}.") + logging.error(f"The operation {key} has NOT finished successfully, because of {problem}.") result = 1 self.store_results(key, result) return result def in_output(self, text, what): + """Checks that the text is in the DNF output.""" if isinstance(text, str): text = text.split("\n") elif isinstance(text, list): @@ -281,6 +329,7 @@ class TestSuite: if not self.whitelist: whitelist = [] else: + logging.info("Module whitelist found. It will be used for some tests.") try: with open(self.whitelist) as f: wlist = f.readlines() @@ -295,15 +344,29 @@ class TestSuite: return whitelist def set_fake(self, fake=False, releasever=None): + """Switches the fake environment on.""" + # If you use this in the test method, the operation will run in test environment. Otherwise, it will run + # for real on the host system data. You can also switch the test environment on using -d (--dryrun) when + # invoking the script on CLI. if fake == 'true' or fake == 'True': + print("======== The test will run in fake environment.========") + logging.info("Fake environment has been required for this test.") self.fake = True else: + print("======== The test will run on the real host packages.========") + logging.info("Test is running with real system data.") self.fake = False + self.install = "--installroot=/testinstall" self.releasever = releasever + self.relver = f"--releasever={self.releasever}" + self.extras1 = f"--setopt=module_platform_id=platform:f{self.releasever}" + self.extras2 = f"--setopt=tsflags=justdb" + return 0 class ModuleTest: + # These are various test methods that do the actual testing. def __init__(self, suite, scenario): self.suite = suite self.scenario = scenario @@ -322,13 +385,17 @@ class ModuleTest: } def results(self): + """Return the overall results.""" return self.overall def decode_error(self, code): + """Decode the exit error code.""" + # We do not use it, yet, I suppose. return self.errorcodes[code] def list_module(self, module, stream=None, profile=None): + """Provide information on the specified module.""" print('-------- List module ---------') key = f"{module}:{stream}/{profile}" modules = self.suite.module_list() @@ -363,21 +430,28 @@ class ModuleTest: def check_install(self, module, stream): + """Check that a module is installed.""" print('-------- Checking for module in list --------') key = f"{module}:{stream}" res1 = self.suite.is_listed(module, stream, 'installed') if res1[key] == 'pass': self.overall['checkinstall'] = 'pass' logging.info(f"The result of {key} is PASS.") + return 0 else: self.overall['checkinstall'] = 'fail' logging.info(f"The result of {key} is FAIL.") + return 1 print(' ') def enable_module(self, module, stream, profile = None): + """Test that a module can be enabled.""" print('-------- Enable module ---------') key = f"{module}:{stream}" + # A module gets enabled, when the DNF operation is successful, + # when the module is shown in the --enabled list and not shown + # in the --disabled list. res1 = self.suite.use_module(module, 'enable', stream, profile) print(f"DNF enables {key} => {res1}") res2 = self.suite.is_listed(module, stream, profile, 'enabled') @@ -387,6 +461,7 @@ class ModuleTest: if res1 == 0 and res2 == 0 and res3 == 111: self.overall['enable'] = 'pass' logging.info(f"The result of enabling {key} is PASS.") + return 0 else: problems = [res1, res2, res3] for p in problems: @@ -394,12 +469,15 @@ class ModuleTest: if self.fail == 'hard': self.overall['enable'] = 'fail' logging.info(f"The result of enabling {key} is FAIL.") + return 1 else: self.overall['enable'] = 'soft' logging.info(f"The result of enabling {key} is SOFTFAIL.") + return 2 print('') def disable_module(self, module, stream, profile = None): + """Test that a module can be disabled.""" print('-------- Disable module ---------') if self.newer != 'dummy': stream = self.newer @@ -407,6 +485,9 @@ class ModuleTest: key = f"{module}:{stream}" else: key = f"{module}" + # A module gets disabled, when the DNF operation is successful, + # when the module is not shown in the --enabled list and shown + # in the --disabled list. res1 = self.suite.use_module(module, 'disable', stream) print(f"DNF disables {key} =>", res1) res2 = self.suite.is_listed(module, stream, profile, 'disabled') @@ -418,6 +499,7 @@ class ModuleTest: if res1 == 0 and res2 == 0 and res3 == 111 and res4 == 111: self.overall['disable'] = 'pass' logging.info(f"The result of disabling {key} is PASS.") + return 0 else: problems = [res1, res2, res3, res4] for p in problems: @@ -425,12 +507,15 @@ class ModuleTest: if self.fail == 'hard': self.overall['disable'] = 'fail' logging.info(f"The result of disabling {key} is FAIL. ") + return 1 else: self.overall['disable'] = 'soft' logging.info(f"The result of disabling {key} is SOFTFAIL.") + return 2 print('') def install_module(self, module, stream=None, profile=None): + """Test that a module can be installed.""" print('-------- Install module ---------') if stream != None and profile != None: key = f"{module}:{stream}/{profile}" @@ -438,6 +523,8 @@ class ModuleTest: key = f"{module}:{stream}" else: key = f"{module}" + # It is installed, when the DNF operation finishes withou errors, + # when it is listen in --enabled and --installed, but not in --disabled. res1 = self.suite.use_module(module, 'install', stream, profile) print(f"DNF installs {key} =>", res1) res2 = self.suite.is_listed(module, stream, profile, 'installed') @@ -449,16 +536,20 @@ class ModuleTest: if res1 == 0 and res2 == 0 and res3 == 0 and res4 == 111: self.overall['install'] = 'pass' logging.info(f"The result of installing {key} is PASS.") + return 0 else: if self.fail == 'hard': self.overall['install'] = 'fail' logging.info(f"The result of installing {key} is FAIL.") + return 1 else: self.overall['install'] = 'soft' logging.info(f"The result of installing {key} is SOFTFAIL.") + return 2 print('') def remove_module(self, module, stream=None, profile=None): + """Tests that a module can be removed.""" print('-------- Remove module ---------') if self.newer != 'dummy': stream = self.newer @@ -466,6 +557,8 @@ class ModuleTest: key = f"{module}:{stream}" else: key = f"{module}" + # It can be removed, when the DNF operation finishes without error + # and when it not listed in --installed. res1 = self.suite.use_module(module, 'remove', stream, profile) print(f"DNF removes {key} =>", res1) res2 = self.suite.is_listed(module, stream, profile, 'installed') @@ -473,13 +566,16 @@ class ModuleTest: if res1 == 0 and res2 == 111: self.overall['remove'] = 'pass' logging.info(f"The result of removing {key} is PASS.") + return 0 else: if self.fail == 'hard': self.overall['remove'] = 'fail' logging.info(f"The result of removing {key} is FAIL.") + return 1 else: self.overall['remove'] = 'soft' logging.info(f"The result of removing {key} is SOFTFAIL.") + return 2 print('') def reset_module(self, module, stream=None, profile=None): @@ -495,7 +591,6 @@ class ModuleTest: res1 = self.suite.is_listed(module, stream, profile, 'enabled') res2 = self.suite.is_listed(module, stream, profile, 'disabled') if res1 == 0 or res2 == 0: - logging.info('The module name has been found in --enabled or --disabled.') res3 = self.suite.use_module(module, 'reset', stream, profile) print(f"DNF resets {module} => {res3}") res1 = self.suite.is_listed(module, stream, profile, 'enabled') @@ -505,9 +600,11 @@ class ModuleTest: if res3 == 0 and res1 == 111 and res2 == 111: self.overall['reset'] = 'pass' logging.info(f"The result of resetting {module} is PASS.") + return 0 else: self.overall['reset'] = 'fail' logging.info(f"The result of resetting {module} is FAIL.") + return 1 else: print('It seems that the module does not exist or is already reset.') @@ -516,7 +613,7 @@ class ModuleTest: def switch_stream(self, module, oldstr, newstr): """Tests switching the stream""" - # Currently this method is not used, because it is outdated. + # Currently this method is not used, because it is outdated. print('-------- Switching streams ---------') oldkey = f"{module}:{oldstr}" newkey = f"{module}:{newstr}" @@ -556,16 +653,27 @@ class ModuleTest: varieties = modules[module] rstream = [] rprofile = [] + # Originally, this was written to check whether a module has the defaults (stream and profile) + # correctly set. Whitelisted modules were not tested because they did not have the defaults on purpose. + # However, it has been decided that only if there is no default profile set, the test should fail. + # Non default profiles are correct, because the maintainer do not want modular content precede the + # ursine content. for variety in varieties: keys = variety.keys() if 'default-stream' in keys: rstream.append('pass') else: + # When there is no default stream defined, we only should test that this is a wanted + # behaviour by checking the particular yaml file in the defaults git repository. + # Such test will also be added later. Until then, you will be warned to check that + # repository, when a module without default stream is found. rstream.append('check') if 'default-profile' in keys: rprofile.append('pass') else: + # When no default profile is set, fail immediately, because such module cannot be + # installed using `dnf module install module:stream`, which is a requirement. rprofile.append('fail') result = [] if 'pass' not in rstream: @@ -584,32 +692,66 @@ class ModuleTest: result.append('multi') totalresults[module] = result - #print(json.dumps(totalresults, sort_keys=True, indent=4)) logging.info(json.dumps(totalresults, sort_keys=True)) problems = 0 + print("======== Showing only problematic modules. ========") for p in totalresults.keys(): - print(f"Testing {p}: {totalresults[p]}") if 'pass' not in totalresults[p]: + print(f"Testing {p}: {totalresults[p]}") problems += 1 - print(f"There were altogether {problems} incomplete modules.") + print(f"\nThere were altogether {problems} incomplete modules.") if problems == 0: self.overall['checkdefaults'] = 'pass' logging.info(f"There were no errors in module stream and profile definitions found.") + return 0 else: - #self.overall['checkdefaults'] = 'fail' - # Until we know exactly how to treat the failures, do not report failure. - self.overall['checkdefaults'] = 'softfail' + self.overall['checkdefaults'] = 'fail' logging.info(f"There were {problems} problems found in module stream and profile definitions.") - - def test_all(self): + return 1 + + def install_all(self): + """Install all modules, all their streams and profiles. One after another.""" + # This tests all available modules, if they can be installed. As this uses + # the fake environment which is newly prepared before each module is + # installed, it downloads all the dnf metadata and therefore takes + # quite a long time. modules = self.suite.module_list() - print(f"{len(self.suite.modlist)} modules will be tested.") + self.suite.fake = True + total = len(self.suite.modlist) + if total == 0: + total = 1 + partial = 0 + print(f"{total} modules will be tested.") + prevname = '' for name in modules.keys(): - print(f"Testing module {name}.") + completed = round((partial/total)*100) + variants = self.suite.module_info(name) + streams = variants.keys() + for stream in streams: + profiles = variants[stream] + for profile in profiles: + profile = profile.strip() + print("--------------------------------------------------") + print(f"Testing {name}:{stream}/{profile} ({completed}% done)") + if name != prevname: + prevname = name + partial += 1 + self.suite.clean_installation() + retcode = self.install_module(name, stream, profile) + key = f"{name}:{stream}/{profile}" + if retcode == 0: + self.overall[key] = 'pass' + else: + self.overall[key] = 'fail' + def run_test(self, module, fail, newer, stream, profile): + """Run a particular test.""" + # This runs the test assigned to an action. An action, can be selected on the CLI + # using the -a (--action) switch. Multiple actions can be specified, for example: + # -a enable,install,remove,disable,reset self.newer = newer self.older = stream self.fail = fail @@ -642,11 +784,12 @@ class ModuleTest: print(json.dumps(info, sort_keys=True, indent=4)) elif s == 'checkdefaults': self.find_no_defaults() - elif s == 'testall': - self.test_all() + elif s == 'allinstall': + self.install_all() class Parser: def __init__(self): + """Inititate the CLI control.""" self.parser = argparse.ArgumentParser() self.parser.add_argument('-m', '--module', default='testmodule', help='The name of the module you wish to work with.') self.parser.add_argument('-s', '--stream', default=None, help='The name of the stream you want to use.') @@ -658,7 +801,8 @@ class Parser: self.parser.add_argument('-w', '--whitelist', default=None, help='File with modules to be skipped when testing default streams and profiles.') self.parser.add_argument('-r', '--releasever', default=None, help='Sets the release version number for fake installation test (use only when you want to invoke mass instalation).') self.parser.add_argument('-d', '--dryrun', default=False, help='Switches dry run when installing modules.') - self.parser.add_argument('--everything', default=False, help='Run all tests on all modules in the system.') + #self.parser.add_argument('--everything', default=False, help='Run all tests on all modules in the system.') + def return_args(self): args = self.parser.parse_args() return args @@ -676,7 +820,7 @@ if __name__ == '__main__': log = args['log'] whitelist = args['whitelist'] releasever = args['releasever'] - everything = args['everything'] + #everything = args['everything'] fake = args['dryrun'] numloglevel = getattr(logging, log.upper()) @@ -685,6 +829,11 @@ if __name__ == '__main__': dnf = DNFoutput() suite = TestSuite(dnf, whitelist) + # When releasever and fake are not set, it will send the False value + # and the fake environment will not be switched on. + # Release version is important for the installroot, it will not install anything without it. + # The correct releasver should be taked from environmental variables, if you want to use + # this script in OpenQA or some others CIs. suite.set_fake(fake, releasever) test = ModuleTest(suite, action)