From 05560ba1cabcec0f252abb9eb6b44017817b7395 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Aug 06 2019 16:58:42 +0000 Subject: Fix tabs vs. spaces, fix whitespace errors, rewrap a comment Tabs vs. spaces caused the script to fail to run, whitespace issues and inconsistencies are just cosmetic. PEP-8 requires comments to be max 72 characters wide; bit wider is usually OK but these lines were ridiculously long. Signed-off-by: Adam Williamson --- diff --git a/modular_functions.py b/modular_functions.py index 61353c2..fc9cd44 100755 --- a/modular_functions.py +++ b/modular_functions.py @@ -8,7 +8,7 @@ Created on Thu Aug 30 14:30:00 2018 This script uses the dnf to do various operations on modular packages and reports such operations -were succesfull from the users' perspective. It does not test the sanity of the modular content, +were succesfull from the users' perspective. It does not test the sanity of the modular content, or any correctness of installed or removed RPM packages. @licence: GPLv3 @@ -26,6 +26,7 @@ import subprocess import sys import yaml + class DNFoutput: # This class uses the DNF code to communicate with DNF in order to get a correct and parsed list of all modules to further processing. def __init__(self): @@ -80,7 +81,7 @@ class DNFoutput: modulePackage = active[0] else: modulePackage = nameStreamArch[0] - + default_str, enabled_str, disabled_str = self._module_strs_formater(modulePackage) default_profiles = self.base._moduleContainer.getDefaultProfiles(modulePackage.getName(), modulePackage.getStream()) profiles_str = self._profile_report_formater(modulePackage, default_profiles, enabled_str) @@ -92,14 +93,13 @@ class DNFoutput: "summary": modulePackage.getSummary() } ) - + return modules def output(self): return self._create_and_fill_table(self.latest) - class TestSuite: def __init__(self, dnf, whitelist=None, test='all'): self.dnf = dnf @@ -126,11 +126,10 @@ class TestSuite: else: self.results[key] = [value] - logging.debug('The store_results method has stored the results into self.results.') - - + logging.debug('The store_results method has stored the results into self.results.') + def module_list(self): - # Get the modular data from the DNF. It comes as a list of dictionaries. + # Get the modular data from the DNF. It comes as a list of dictionaries. processed = self.dnf.output() # Let us parse the data and for each module create a dictionary, where name is the key and the rest comes as a value. @@ -166,22 +165,22 @@ class TestSuite: module['profiles'] = nprofiles info = line['summary'] module['info'] = info - + # Now, let us make a database (dict) with all the modules, with name as keys and list of dicts # for various streams as values. if name not in modulelist.keys(): #If this is first occurence modulelist[name] = [module] else: # If this is another occurence of same module with different streams. - if module not in modulelist[name]: + if module not in modulelist[name]: modulelist[name].append(module) else: pass - + self.modlist = modulelist - + logging.debug('The "dnf module list" operation was succesful. Data have been obtained from the DNF.') return self.modlist - + def module_info(self, moduleName): """Returns a dictionary with streams as keys and profiles as values.""" modules = self.module_list() @@ -192,7 +191,6 @@ class TestSuite: logging.debug("The module_info method ran successfully.") return info - def use_module(self, module, operation, stream=None, profile=None): """This method calls different modular commands.""" # Each time, we need to call a modular dnf command, we will call this method @@ -232,12 +230,12 @@ class TestSuite: logging.error(f"The operation {key} has NOT finished successfully, because of {problem}.") print('There was a DNF problem! Check the log for further information.') result = 1 - self.store_results(f'dnf module {operation}', result) + self.store_results(f'dnf module {operation}', result) return result def clean_installation(self): """Cleans installation data in fake environment.""" - # All fake tests run in installroot=/testinstall, before each test, the installroot + # All fake tests run in installroot=/testinstall, before each test, the installroot # should be deleted to achieve a clean test environment. raw = subprocess.run(['rm', '-rf', '/testinstall'], capture_output=True) if raw.returncode == 0: @@ -247,9 +245,7 @@ class TestSuite: print("\n======== There was a problem deleting the test environment, because it had been already deleted or never existed. ========\n") raw = raw.stderr.decode('utf-8') logging.error(f"The test date have not been wiped succesfully: {raw} ") - - def is_listed(self, module, stream, profile, inlist): """Returns if the specified module is listed in dnf module list.""" option = f"--{inlist}" @@ -270,7 +266,7 @@ class TestSuite: if raw.returncode == 0: raw = raw.stdout.decode('utf-8') self.outputs[option] = raw - raw = raw.split("\n") + raw = raw.split("\n") modfound = 100 strfound = 10 profound = 1 @@ -282,14 +278,14 @@ class TestSuite: elif stream == None: strfound = 0 if profile != None and profile in line: - profound = 0 + profound = 0 elif profile == None: profound = 0 result = modfound + strfound + profound if result == 0: break # The result show a 1 for every error. Therefore, it can be easily spotted, where the problem is - # even if you do now read the log file. + # even if you do now read the log file. # 0 - everything OK # 1 - the specified profile is not in list, but the name and stream are # 10 - the specified stream is not in list, but the name and profile are @@ -301,7 +297,7 @@ class TestSuite: problem = raw.stderr.decode('utf-8') logging.error(f"The operation {key} has NOT finished successfully, because of {problem}.") result = 1 - self.store_results(key, result) + self.store_results(key, result) return result def in_output(self, text, what): @@ -324,10 +320,10 @@ class TestSuite: else: return (0, c) logging.debug('The string was NOT found in the DNF output.') - + def load_whitelist(self): """Reads the file with whitelisted modules and returns the content.""" - # Some modules do not have the default streams and/or profiles set on purpose. + # Some modules do not have the default streams and/or profiles set on purpose. # Their names can be load into the suite to prevent them from being reported. if not self.whitelist: whitelist = [] @@ -339,7 +335,7 @@ class TestSuite: whitelist = [] for m in wlist: whitelist.append(m.strip()) - + except FileNotFoundError: print(f"File {self.whitelist} has not been found. Whitelist could not be read, proceeding without it.") logging.error(f"File {self.whitelist} has not been found. Proceeding without it.") @@ -348,7 +344,7 @@ class TestSuite: def set_fake(self, fake=False, releasever=None): """Switches the fake environment on.""" - # If you use this in the test method, the operation will run in test environment. Otherwise, it will run + # If you use this in the test method, the operation will run in test environment. Otherwise, it will run # for real on the host system data. You can also switch the test environment on using -d (--dryrun) when # invoking the script on CLI. if fake == 'true' or fake == 'True': @@ -387,7 +383,8 @@ class TestSuite: print(f"{key}") logging.error(f"The {key} seems not to exist for {module}.") return None - + + class ModuleTest: # These are various test methods that do the actual testing. def __init__(self, suite, scenario): @@ -399,7 +396,7 @@ class ModuleTest: self.fail = 'hard' self.whitelist = [] self.errorcodes = { - 0: 'Everything went ok in this part.', + 0: 'Everything went ok in this part.', 1: 'A problem in interaction with DNF occured.', 100: 'The module stream was not found in the selected list.', 110: 'The module stream was not found in the selected list.', @@ -416,7 +413,6 @@ class ModuleTest: # We do not use it, yet, I suppose. return self.errorcodes[code] - def list_module(self, module, stream=None, profile=None): """Provide information on the specified module.""" print('-------- List module ---------') @@ -443,7 +439,7 @@ class ModuleTest: except KeyError: print(f"Default profile: none") print('------------------------------') - logging.info("The list method finished successfully.") + logging.info("The list method finished successfully.") return(info) else: print('Module does not exist.') @@ -451,7 +447,6 @@ class ModuleTest: self.overall['list'] = 'fail' return None - def check_install(self, module, stream): """Check that a module is installed.""" print('-------- Checking for module in list --------') @@ -467,7 +462,6 @@ class ModuleTest: return 1 print(' ') - def enable_module(self, module, stream, profile = None): """Test that a module can be enabled.""" print('-------- Enable module ---------') @@ -536,7 +530,7 @@ class ModuleTest: logging.info(f"The result of disabling {key} is SOFTFAIL.") return 2 print('') - + def install_module(self, module, stream=None, profile=None): """Test that a module can be installed.""" print('-------- Install module ---------') @@ -576,7 +570,7 @@ class ModuleTest: print('-------- Remove module ---------') if self.newer != 'dummy': stream = self.newer - if stream != None: + if stream != None: key = f"{module}:{stream}" else: key = f"{module}" @@ -608,7 +602,7 @@ class ModuleTest: # disabling an enabled module will not bring it to its neutral state (and vice versa). To bring the # module into a neutral state, one has to reset it. Resetting a module is an important part of # module stream switching, if one decides to do so. - # To reset a module, we will check that it is either enabled, or disabled (that means, not neutral, then + # To reset a module, we will check that it is either enabled, or disabled (that means, not neutral, then # we reset it and check that it is not enabled nor disabled -> neutral. print('-------- Resetting module ---------') res1 = self.suite.is_listed(module, stream, profile, 'enabled') @@ -636,19 +630,22 @@ class ModuleTest: def switch_stream(self, module, oldstr, newstr): """Tests switching the stream""" - # Direct stream switching is not allowed any more. To switch streams, it is required to remove the current module and stream, reset it and - # install the same module with a different stream again. This action uses other (predefined) actions to achieve it. Same functionality - # can be achieved by using two different tests. - + # Direct stream switching is not allowed any more. To switch + # streams, it is required to remove the current module and + # stream, reset it and install the same module with a different + # stream again. This action uses other (predefined) actions to + # achieve it. Same functionality can be achieved by using two + # different tests. + print('-------- Switching streams ---------') oldkey = f"{module}:{oldstr}" newkey = f"{module}:{newstr}" - res1 = self.suite.use_module(module, oldstr, 'install') - res2 = self.suite.use_module(module, oldstr, 'remove') - res3 = self.suite.use_module(module, oldstr, 'reset') - res4 = self.suite.use_module(module, newstr, 'install') - + res1 = self.suite.use_module(module, oldstr, 'install') + res2 = self.suite.use_module(module, oldstr, 'remove') + res3 = self.suite.use_module(module, oldstr, 'reset') + res4 = self.suite.use_module(module, newstr, 'install') + if res1 == 0 and res2 == 0 and res3 == 0 and res4 == 0: self.overall['switch'] = 'pass' logging.info(f"The result of switching {oldkey} for {newkey} is PASS.") @@ -700,11 +697,11 @@ class ModuleTest: result = [] if 'pass' not in rstream: if 'check' in rstream: - result.append('check yaml for stream definition') + result.append('check yaml for stream definition') else: - result.append('stream') + result.append('stream') if 'pass' not in rprofile: - result.append('profile') + result.append('profile') if 'stream' not in result and 'profile' not in result: result.append('pass') if module in whitelist: @@ -713,9 +710,9 @@ class ModuleTest: if len(varieties) > 1: result.append('multi') - totalresults[module] = result + totalresults[module] = result logging.info(json.dumps(totalresults, sort_keys=True)) - + problems = 0 print("======== Showing only problematic modules. ========") for p in totalresults.keys(): @@ -762,7 +759,7 @@ class ModuleTest: if name != prevname: prevname = name partial += 1 - self.suite.clean_installation() + self.suite.clean_installation() retcode = self.install_module(name, stream, profile) key = f"{name}:{stream}/{profile}" if retcode == 0: @@ -774,8 +771,8 @@ class ModuleTest: def testyaml(self): """Test the sanity of the module yaml file.""" - # This makes sure that there is yaml file defined for every module in the system defined - # and published in https://pagure.io/releng/fedora-module-defaults.git, and that the yaml + # This makes sure that there is yaml file defined for every module in the system defined + # and published in https://pagure.io/releng/fedora-module-defaults.git, and that the yaml # files do have all their requirements set. modules = self.suite.module_list() self.suite.create_yamldb('./yaml/') @@ -834,14 +831,14 @@ class ModuleTest: else: if module not in whitelist: results.append('fail') - + if parsed_results: print(f"There are problems with some of the yaml files. Check logs.") logging.error(f"Problems with yaml files found.") self.overall['yaml_sanity'] = 'fail' else: self.overall['yaml_sanity'] = 'pass' - + if 'fail' in results: self.overall['yaml_exists'] = 'fail' else: @@ -889,6 +886,7 @@ class ModuleTest: elif s == 'allinstall': self.install_all() + class Parser: def __init__(self): """Inititate the CLI control.""" @@ -909,6 +907,7 @@ class Parser: args = self.parser.parse_args() return args + if __name__ == '__main__': options = Parser() args = options.return_args() @@ -928,7 +927,7 @@ if __name__ == '__main__': logging.basicConfig(filename='modular.log', filemode='w', level=numloglevel) logging.info('Script started.') - + dnf = DNFoutput() suite = TestSuite(dnf, whitelist) # When releasever and fake are not set, it will send the False value @@ -936,8 +935,8 @@ if __name__ == '__main__': # Release version is important for the installroot, it will not install anything without it. # The correct releasver should be taked from environmental variables, if you want to use # this script in OpenQA or some others CIs. - suite.set_fake(fake, releasever) - + suite.set_fake(fake, releasever) + test = ModuleTest(suite, action) test.run_test(module, fail, newstream, oldstream, profile) @@ -955,6 +954,3 @@ if __name__ == '__main__': else: logging.info('Script finished with exit code 0.') sys.exit(0) - - -