From 9c5e6650ed581347023d28e86cca0b485d50919e Mon Sep 17 00:00:00 2001 From: Petr Sklenar Date: Jun 05 2017 12:52:20 +0000 Subject: [PATCH 1/2] code clean up --- diff --git a/moduleframework/common.py b/moduleframework/common.py index c242999..d0a494f 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -33,9 +33,11 @@ import socket import os import linecache + class ModuleFrameworkException(Exception): - def __init__(self, *args,**kwargs): - super(ModuleFrameworkException, self).__init__('EXCEPTION MTF: ', *args,**kwargs) + def __init__(self, *args, **kwargs): + super(ModuleFrameworkException, self).__init__( + 'EXCEPTION MTF: ', *args, **kwargs) exc_type, exc_obj, tb = sys.exc_info() if tb is not None: f = tb.tb_frame @@ -45,29 +47,35 @@ class ModuleFrameworkException(Exception): line = linecache.getline(filename, lineno, f.f_globals) print "-----------\n| EXCEPTION IN: {} \n| LINE: {}, {} \n| ERROR: {}\n-----------".format(filename, lineno, line.strip(), exc_obj) + class NspawnExc(ModuleFrameworkException): - def __init__(self,*args,**kwargs): - super(NspawnExc, self).__init__('TYPE nspawn', *args,**kwargs) + def __init__(self, *args, **kwargs): + super(NspawnExc, self).__init__('TYPE nspawn', *args, **kwargs) + class RpmExc(ModuleFrameworkException): - def __init__(self,*args,**kwargs): - super(RpmExc, self).__init__('TYPE rpm', *args,**kwargs) + def __init__(self, *args, **kwargs): + super(RpmExc, self).__init__('TYPE rpm', *args, **kwargs) + class ContainerExc(ModuleFrameworkException): - def __init__(self,*args,**kwargs): - super(ContainerExc, self).__init__('TYPE container', *args,**kwargs) + def __init__(self, *args, **kwargs): + super(ContainerExc, self).__init__('TYPE container', *args, **kwargs) + class ConfigExc(ModuleFrameworkException): - def __init__(self,*args,**kwargs): - super(ConfigExc, self).__init__('TYPE config', *args,**kwargs) + def __init__(self, *args, **kwargs): + super(ConfigExc, self).__init__('TYPE config', *args, **kwargs) + class PDCExc(ModuleFrameworkException): - def __init__(self,*args,**kwargs): - super(PDCExc, self).__init__('TYPE PDC', *args,**kwargs) + def __init__(self, *args, **kwargs): + super(PDCExc, self).__init__('TYPE PDC', *args, **kwargs) + class KojiExc(ModuleFrameworkException): - def __init__(self,*args,**kwargs): - super(KojiExc, self).__init__('TYPE Koji', *args,**kwargs) + def __init__(self, *args, **kwargs): + super(KojiExc, self).__init__('TYPE Koji', *args, **kwargs) defroutedev = netifaces.gateways().get('default').values( @@ -100,18 +108,21 @@ PDCURL = "https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants" REPOMD = "repodata/repomd.xml" MODULEFILE = 'tempmodule.yaml' # default value of process timeout in sec -DEFAULTPROCESSTIMEOUT = 2*60 +DEFAULTPROCESSTIMEOUT = 2 * 60 DEFAULTRETRYCOUNT = 3 # time in seconds DEFAULTRETRYTIMEOUT = 30 DEFAULTNSPAWNTIMEOUT = 10 + def is_debug(): return bool(os.environ.get("DEBUG")) + def is_not_silent(): return not is_debug() + def print_info(*args): """ Print data to selected output in case you are not in testing class, there is self.log @@ -130,6 +141,7 @@ def print_info(*args): trans_dict) print >> sys.stderr, out + def print_debug(*args): """ Print data to selected output in case you are not in testing class, there is self.log diff --git a/moduleframework/generator.py b/moduleframework/generator.py index 845da59..8de5006 100755 --- a/moduleframework/generator.py +++ b/moduleframework/generator.py @@ -66,7 +66,9 @@ class GeneratedTestsConfig(module_framework.AvocadoTest): for line in testlines: # only use shell=True for runHost() calls, otherwise variables etc. # get expanded too early, i.e. on the host - self.output = self.output + ' self.%s(""" %s """, shell=%r)\n' % (method, line, method == "runHost") + self.output = self.output + \ + ' self.%s(""" %s """, shell=%r)\n' % ( + method, line, method == "runHost") print("Added test (runmethod: %s): %s" % (method, testname)) diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 0294c09..042efa4 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -49,6 +49,7 @@ import warnings PROFILE = None + def skipTestIf(value, text="Test not intended for this module profile"): """ function what solves troubles that it is not possible to call SKIP inside code @@ -59,7 +60,8 @@ def skipTestIf(value, text="Test not intended for this module profile"): :return: None """ if value: - raise ModuleFrameworkException("DEPRECATED, don't use this skip, use self.cancel() inside test function, or self.skip() in setUp()") + raise ModuleFrameworkException( + "DEPRECATED, don't use this skip, use self.cancel() inside test function, or self.skip() in setUp()") class CommonFunctions(object): @@ -88,7 +90,10 @@ class CommonFunctions(object): try: formattedcommand = command.format(**trans_dict) except KeyError: - raise ModuleFrameworkException("Command is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ or }}, possible values in trans_dict are:", trans_dict) + raise ModuleFrameworkException( + "Command is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ " + "or }}, possible values in trans_dict are:", + trans_dict) return utils.process.run("%s" % formattedcommand, **kwargs) def installTestDependencies(self, packages=None): @@ -134,7 +139,7 @@ class CommonFunctions(object): except ValueError: pass - def getPackageList(self,profile=None): + def getPackageList(self, profile=None): """ Return list of packages what has to be installed inside module @@ -150,16 +155,20 @@ class CommonFunctions(object): for x in self.config['packages'].get('profiles') if self.config[ 'packages'].get('profiles') else []: packages_profiles = packages_profiles + \ - self.getModulemdYamlconfig()['data']['profiles'][x]['rpms'] + self.getModulemdYamlconfig()[ + 'data']['profiles'][x]['rpms'] out += packages_rpm + packages_profiles - elif self.getModulemdYamlconfig()['data'].get('profiles') and self.getModulemdYamlconfig()['data']['profiles'].get(get_correct_profile()): - out += self.getModulemdYamlconfig()['data']['profiles'][get_correct_profile()]['rpms'] + elif self.getModulemdYamlconfig()['data'].get('profiles') and self.getModulemdYamlconfig()['data'][ + 'profiles'].get(get_correct_profile()): + out += self.getModulemdYamlconfig( + )['data']['profiles'][get_correct_profile()]['rpms'] else: # fallback solution when it is not known what to install out.append("bash") else: - out += self.getModulemdYamlconfig()['data']['profiles'][profile]['rpms'] + out += self.getModulemdYamlconfig( + )['data']['profiles'][profile]['rpms'] print_info("PCKGs to install inside module:", out) return out @@ -178,7 +187,7 @@ class CommonFunctions(object): link = cconfig elif not get_if_module(): trans_dict["GUESTPACKAGER"] = "yum -y" - link = {"data":{}} + link = {"data": {}} else: if self.config is None: self.loadconfig() @@ -192,7 +201,6 @@ class CommonFunctions(object): except IOError as e: raise ConfigExc("Cannot load file") - def getIPaddr(self): """ Return ip addr string of guest machine @@ -214,7 +222,7 @@ class ContainerHelper(CommonFunctions): """ set basic object variables """ - super(ContainerHelper,self).__init__() + super(ContainerHelper, self).__init__() self.loadconfig() self.info = self.config['module']['docker'] self.tarbased = None @@ -285,7 +293,8 @@ class ContainerHelper(CommonFunctions): :return: None """ if not os.path.isfile('/usr/bin/docker-current'): - self.runHost("{HOSTPACKAGER} install docker",verbose=is_not_silent()) + self.runHost("{HOSTPACKAGER} install docker", + verbose=is_not_silent()) def __prepareContainer(self): """ @@ -293,7 +302,7 @@ class ContainerHelper(CommonFunctions): :return: None """ - if self.tarbased == False and self.jmeno == self.icontainer and "docker.io" not in self.info[ + if self.tarbased is False and self.jmeno == self.icontainer and "docker.io" not in self.info[ 'container']: registry = re.search("([^/]*)", self.icontainer).groups()[0] if registry not in open('/etc/sysconfig/docker', 'rw').read(): @@ -317,7 +326,8 @@ class ContainerHelper(CommonFunctions): elif "docker=" in self.icontainer: pass else: - self.runHost("docker pull %s" % self.jmeno, verbose=is_not_silent()) + self.runHost("docker pull %s" % + self.jmeno, verbose=is_not_silent()) self.containerInfo = json.loads( self.runHost( @@ -336,7 +346,9 @@ class ContainerHelper(CommonFunctions): if 'start' in self.info and self.info['start']: self.docker_id = self.runHost( "%s -d %s" % - (self.info['start'], self.jmeno), shell=True, ignore_bg_processes=True, verbose=is_not_silent()).stdout + (self.info['start'], + self.jmeno), shell=True, ignore_bg_processes=True, + verbose=is_not_silent()).stdout else: self.docker_id = self.runHost( "docker run %s %s %s" % @@ -350,17 +362,23 @@ class ContainerHelper(CommonFunctions): ignore_status=True, verbose=False) b = self.run( "%s install %s" % - (trans_dict["GUESTPACKAGER"]," ".join( + (trans_dict["GUESTPACKAGER"], " ".join( self.getPackageList())), ignore_status=True, verbose=False) if a.exit_status == 0: - print_info("Packages installed via {HOSTPACKAGER}", a.stdout) + print_info( + "Packages installed via {HOSTPACKAGER}", a.stdout) elif b.exit_status == 0: - print_info("Packages installed via {GUESTPACKAGER}", b.stdout) + print_info( + "Packages installed via {GUESTPACKAGER}", b.stdout) else: - print_info("Nothing installed (nor via {HOSTPACKAGER} nor {GUESTPACKAGER}), but package list is not empty", self.getPackageList()) + print_info( + "Nothing installed (nor via {HOSTPACKAGER} nor {GUESTPACKAGER}), but package list is not empty", + self.getPackageList()) if self.status() is False: - raise ContainerExc("Container %s (for module %s) is not running, probably DEAD immediately after start (ID: %s)" % (self.jmeno, self.moduleName, self.docker_id)) + raise ContainerExc( + "Container %s (for module %s) is not running, probably DEAD immediately after start (ID: %s)" % ( + self.jmeno, self.moduleName, self.docker_id)) def stop(self): """ @@ -370,8 +388,10 @@ class ContainerHelper(CommonFunctions): """ if self.status(): try: - self.runHost("docker stop %s" % self.docker_id, verbose=is_not_silent()) - self.runHost("docker rm %s" % self.docker_id, verbose=is_not_silent()) + self.runHost("docker stop %s" % + self.docker_id, verbose=is_not_silent()) + self.runHost("docker rm %s" % + self.docker_id, verbose=is_not_silent()) except Exception as e: print_debug(e, "docker already removed") pass @@ -412,7 +432,8 @@ class ContainerHelper(CommonFunctions): :return: None """ self.start() - self.runHost("docker cp %s %s:%s" % (src, self.docker_id, dest), verbose=is_not_silent()) + self.runHost("docker cp %s %s:%s" % + (src, self.docker_id, dest), verbose=is_not_silent()) def copyFrom(self, src, dest): """ @@ -423,7 +444,8 @@ class ContainerHelper(CommonFunctions): :return: None """ self.start() - self.runHost("docker cp %s:%s %s" % (self.docker_id, src, dest), verbose=is_not_silent()) + self.runHost("docker cp %s:%s %s" % + (self.docker_id, src, dest), verbose=is_not_silent()) def __callSetupFromConfig(self): """ @@ -432,7 +454,8 @@ class ContainerHelper(CommonFunctions): :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("setup"), shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -441,7 +464,8 @@ class ContainerHelper(CommonFunctions): :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("cleanup"), shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) class RpmHelper(CommonFunctions): @@ -463,14 +487,15 @@ class RpmHelper(CommonFunctions): self.moduleName) self.info = self.config['module']['rpm'] self.repos = [] - self.whattoinstallrpm="" - self.bootstrappackages=[] + self.whattoinstallrpm = "" + self.bootstrappackages = [] def setModuleDependencies(self): temprepositories = {} if self.getModulemdYamlconfig()["data"].get("dependencies") and self.getModulemdYamlconfig()["data"][ - "dependencies"].get("requires"): - temprepositories = self.getModulemdYamlconfig()["data"]["dependencies"]["requires"] + "dependencies"].get("requires"): + temprepositories = self.getModulemdYamlconfig( + )["data"]["dependencies"]["requires"] temprepositories_cycle = dict(temprepositories) for x in temprepositories_cycle: pdc = pdc_data.PDCParser() @@ -504,7 +529,7 @@ class RpmHelper(CommonFunctions): self.__prepare() self.__prepareSetup() - def setRepositoriesAndWhatToInstall(self, repos=[], whattooinstall=[]): + def setRepositoriesAndWhatToInstall(self, repos=None, whattooinstall=None): """ set repositories and packages what to install inside module It can override base usage of this framework to general purpose testing @@ -513,13 +538,16 @@ class RpmHelper(CommonFunctions): :param whattooinstall: list of packages to install inside :return: None """ + if repos is None: + repos = [] alldrepos = [] if repos: - self.repos=repos + self.repos = repos else: if not self.repos: for dep in self.moduledeps: - alldrepos.append(get_latest_repo_url(dep, self.moduledeps[dep])) + alldrepos.append(get_latest_repo_url( + dep, self.moduledeps[dep])) if get_correct_url(): self.repos = [get_correct_url()] + alldrepos elif self.info.get('repo'): @@ -532,8 +560,10 @@ class RpmHelper(CommonFunctions): self.whattoinstallrpm = " ".join(set(whattooinstall)) else: if not self.whattoinstallrpm: - self.bootstrappackages = pdc_data.getBasePackageSet(modulesDict=self.moduledeps, isModule=get_if_module(), isContainer=False) - self.whattoinstallrpm = " ".join(set(self.getPackageList() + self.bootstrappackages)) + self.bootstrappackages = pdc_data.getBasePackageSet(modulesDict=self.moduledeps, + isModule=get_if_module(), isContainer=False) + self.whattoinstallrpm = " ".join( + set(self.getPackageList() + self.bootstrappackages)) def tearDown(self): """ @@ -573,8 +603,10 @@ gpgcheck=0 a = self.runHost( "%s --disablerepo=* --enablerepo=%s* --allowerasing install %s" % - (trans_dict["HOSTPACKAGER"],self.moduleName, self.whattoinstallrpm), ignore_status=True, verbose=is_not_silent()) - b =self.runHost( + (trans_dict["HOSTPACKAGER"], self.moduleName, + self.whattoinstallrpm), ignore_status=True, + verbose=is_not_silent()) + b = self.runHost( "%s --disablerepo=* --enablerepo=%s* --allowerasing distro-sync" % (trans_dict["HOSTPACKAGER"], self.moduleName), ignore_status=True, verbose=is_not_silent()) @@ -594,10 +626,13 @@ gpgcheck=0 """ try: if 'status' in self.info and self.info['status']: - a = self.runHost(self.info['status'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + a = self.runHost( + self.info['status'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) else: - a = self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - print_debug("command:",a.command ,"stdout:",a.stdout, "stderr:", a.stderr) + a = self.runHost("%s" % command, shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) + print_debug("command:", a.command, "stdout:", + a.stdout, "stderr:", a.stderr) return True except BaseException: return False @@ -606,27 +641,29 @@ gpgcheck=0 """ start the RPM based module (like systemctl start service) - :param args: Do not use it directly (It is defined in config.yaml) :param command: Do not use it directly (It is defined in config.yaml) :return: None """ if 'start' in self.info and self.info['start']: - self.runHost(self.info['start'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info['start'], shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) else: - self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost("%s" % command, shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) def stop(self, command="/bin/true"): """ stop the RPM based module (like systemctl stop service) - :param args: Do not use it directly (It is defined in config.yaml) :param command: Do not use it directly (It is defined in config.yaml) :return: None """ if 'stop' in self.info and self.info['stop']: - self.runHost(self.info['stop'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info['stop'], shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) else: - self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost("%s" % command, shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) def run(self, command="ls /", **kwargs): """ @@ -666,7 +703,8 @@ gpgcheck=0 :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("setup"), shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -675,7 +713,8 @@ gpgcheck=0 :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("cleanup"), shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) class NspawnHelper(RpmHelper): @@ -693,6 +732,8 @@ class NspawnHelper(RpmHelper): relative change root path """ super(NspawnHelper, self).__init__() + self.__selinuxState = self.runHost( + "getenforce", ignore_status=True).stdout.strip() time.time() actualtime = time.time() if get_if_do_cleanup(): @@ -718,9 +759,8 @@ class NspawnHelper(RpmHelper): if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): # TODO: workaround because systemd nspawn is now working well in F-25 # (failing because of selinux) - self.__selinuxState = self.runHost( - "getenforce", ignore_status=True).stdout.strip() - self.runHost("setenforce Permissive", ignore_status=True, verbose=is_not_silent()) + self.runHost("setenforce Permissive", + ignore_status=True, verbose=is_not_silent()) self.setModuleDependencies() self.setRepositoriesAndWhatToInstall() self.installTestDependencies() @@ -731,21 +771,25 @@ class NspawnHelper(RpmHelper): def __is_killed(self): for foo in range(DEFAULTRETRYTIMEOUT): time.sleep(foo) - out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) + out = self.runHost("machinectl status %s" % + self.jmeno, verbose=is_debug(), ignore_status=True) if out.exit_status != 0: print_debug("NSPAWN machine %s stopped" % self.jmeno) return True - raise NspawnExc("Unable to stop machine %s within %d" % (self.jmeno,DEFAULTRETRYTIMEOUT)) + raise NspawnExc("Unable to stop machine %s within %d" % + (self.jmeno, DEFAULTRETRYTIMEOUT)) def __is_booted(self): for foo in range(DEFAULTRETRYTIMEOUT): time.sleep(foo) - out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) + out = self.runHost("machinectl status %s" % + self.jmeno, verbose=is_debug(), ignore_status=True) if "logind.service" in out.stdout: time.sleep(2) print_debug("NSPAWN machine %s booted" % self.jmeno) return True - raise NspawnExc("Unable to start machine %s within %d" % (self.jmeno,DEFAULTRETRYTIMEOUT)) + raise NspawnExc("Unable to start machine %s within %d" % + (self.jmeno, DEFAULTRETRYTIMEOUT)) def __prepareSetup(self): """ @@ -757,12 +801,14 @@ class NspawnHelper(RpmHelper): shutil.rmtree(self.chrootpath, ignore_errors=True) os.mkdir(self.chrootpath) try: - self.runHost("machinectl terminate %s" % self.jmeno, verbose=is_debug()) + self.runHost("machinectl terminate %s" % + self.jmeno, verbose=is_debug()) self.__is_killed() except BaseException: pass if not os.path.exists(os.path.join(self.chrootpath, "usr")): - self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent()) + self.runHost("{HOSTPACKAGER} install systemd-container", + verbose=is_not_silent()) repos_to_use = "" counter = 0 for repo in self.repos: @@ -770,11 +816,14 @@ class NspawnHelper(RpmHelper): repos_to_use += " --repofrompath %s%d,%s" % ( self.moduleName, counter, repo) try: - @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT*60, delay=2*60, error=NspawnExc("RETRY: Unable to install packages")) + @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT * 60, delay=2 * 60, + error=NspawnExc("RETRY: Unable to install packages")) def tmpfunc(): self.runHost( "%s install --nogpgcheck --setopt=install_weak_deps=False --installroot %s --allowerasing --disablerepo=* --enablerepo=%s* %s %s" % - (trans_dict["HOSTPACKAGER"], self.chrootpath, self.moduleName, repos_to_use, self.whattoinstallrpm), verbose=is_not_silent()) + (trans_dict["HOSTPACKAGER"], self.chrootpath, self.moduleName, repos_to_use, + self.whattoinstallrpm), verbose=is_not_silent()) + tmpfunc() except Exception as e: raise NspawnExc( @@ -800,8 +849,8 @@ gpgcheck=0 f.write(add) f.close() - # shutil.copy(self.yumrepo, insiderepopath) - # self.runHost("sed s/enabled=0/enabled=1/ -i %s" % insiderepopath, ignore_status=True) + # shutil.copy(self.yumrepo, insiderepopath) + # self.runHost("sed s/enabled=0/enabled=1/ -i %s" % insiderepopath, ignore_status=True) for repo in self.repos: if "file:///" in repo: src = repo[7:] @@ -809,12 +858,14 @@ gpgcheck=0 try: os.makedirs(os.path.dirname(srcto)) except Exception as e: - print_debug(e, "Unable to create DIR (already created)", srcto) + print_debug( + e, "Unable to create DIR (already created)", srcto) pass try: shutil.copytree(src, srcto) except Exception as e: - print_debug(e, "Unable to copy files from:", src, "to:", srcto) + print_debug(e, "Unable to copy files from:", + src, "to:", srcto) pass pkipath = "/etc/pki/rpm-gpg" pkipath_ch = os.path.join(self.chrootpath, pkipath[1:]) @@ -824,7 +875,8 @@ gpgcheck=0 pass for filename in glob.glob(os.path.join(pkipath, '*')): shutil.copy(filename, pkipath_ch) - print_info("repo prepared for microdnf:", insiderepopath, open(insiderepopath, 'r').read()) + print_info("repo prepared for microdnf:", insiderepopath, + open(insiderepopath, 'r').read()) def __bootMachine(self): @@ -854,10 +906,13 @@ gpgcheck=0 """ try: if 'status' in self.info and self.info['status']: - a = self.run(self.info['status'], shell=True, verbose=False, ignore_bg_processes=True) + a = self.run(self.info['status'], shell=True, + verbose=False, ignore_bg_processes=True) else: - a = self.run("%s" % command, shell=True, verbose=False, ignore_bg_processes=True) - print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) + a = self.run("%s" % command, shell=True, + verbose=False, ignore_bg_processes=True) + print_debug("command:", a.command, "stdout:", + a.stdout, "stderr:", a.stderr) return True except BaseException: return False @@ -866,7 +921,6 @@ gpgcheck=0 """ start the RPM based module (like systemctl start service) - :param args: Do not use it directly (It is defined in config.yaml) :param command: Do not use it directly (It is defined in config.yaml) :return: None """ @@ -912,9 +966,9 @@ gpgcheck=0 try: if not kwargs: kwargs = {} - kwargs["verbose"]=is_not_silent() - should_ignore=kwargs.get("ignore_status") - kwargs["ignore_status"]=True + kwargs["verbose"] = is_not_silent() + should_ignore = kwargs.get("ignore_status") + kwargs["ignore_status"] = True b = self.runHost( 'bash -c "cat {chroot}{pin}/stdout; cat {chroot}{pin}/stderr > /dev/stderr; exit `cat {chroot}{pin}/retcode`"'.format( chroot=self.chrootpath, @@ -924,7 +978,7 @@ gpgcheck=0 comout.stdout = b.stdout comout.stderr = b.stderr comout.exit_status = b.exit_status - removesworkaround = re.search('[^(]*\((.*)\)[^)]*',comout.command) + removesworkaround = re.search('[^(]*\((.*)\)[^)]*', comout.command) if removesworkaround: comout.command = removesworkaround.group(1) if comout.exit_status == 0 or should_ignore: @@ -951,7 +1005,7 @@ gpgcheck=0 """ self.runHost( " machinectl copy-to %s %s %s" % - (self.jmeno, src, dest), timeout = DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) + (self.jmeno, src, dest), timeout=DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) def copyFrom(self, src, dest): """ @@ -963,7 +1017,7 @@ gpgcheck=0 """ self.runHost( " machinectl copy-from %s %s %s" % - (self.jmeno, src, dest), timeout = DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) + (self.jmeno, src, dest), timeout=DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) def tearDown(self): """ @@ -972,7 +1026,8 @@ gpgcheck=0 :return: None """ self.stop() - self.runHost("machinectl poweroff %s" % self.jmeno, verbose=is_not_silent()) + self.runHost("machinectl poweroff %s" % + self.jmeno, verbose=is_not_silent()) # self.nspawncont.stop() self.__is_killed() if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): @@ -986,7 +1041,6 @@ gpgcheck=0 shutil.rmtree(self.chrootpath, ignore_errors=True) self.__callCleanupFromConfig() - def __callSetupFromConfig(self): """ Internal method, do not use it anyhow @@ -994,7 +1048,8 @@ gpgcheck=0 :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("setup"), shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -1003,7 +1058,8 @@ gpgcheck=0 :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("cleanup"), shell=True, + ignore_bg_processes=True, verbose=is_not_silent()) # INTERFACE CLASS FOR GENERAL TESTS OF MODULES @@ -1020,8 +1076,9 @@ class AvocadoTest(Test): :avocado: disable """ - def __init__(self,*args, **kwargs): - super(AvocadoTest,self).__init__(*args, **kwargs) + + def __init__(self, *args, **kwargs): + super(AvocadoTest, self).__init__(*args, **kwargs) (self.backend, self.moduleType) = get_correct_backend() self.moduleProfile = get_correct_profile() @@ -1158,7 +1215,8 @@ class AvocadoTest(Test): :return: str """ self.start() - allpackages = self.run(r'rpm -qa --qf="%{{name}}\n"', verbose=is_not_silent()).stdout.split('\n') + allpackages = self.run( + r'rpm -qa --qf="%{{name}}\n"', verbose=is_not_silent()).stdout.split('\n') return allpackages def copyTo(self, *args, **kwargs): @@ -1247,7 +1305,6 @@ class NspawnAvocadoTest(AvocadoTest): super(NspawnAvocadoTest, self).setUp() - def get_correct_backend(): """ Return proper module type, set by config by default_module section, or defined via @@ -1268,7 +1325,8 @@ def get_correct_backend(): elif amodule == 'nspawn': return NspawnHelper(), amodule else: - raise ModuleFrameworkException("Unsupported MODULE={0}".format(amodule), "supproted are: docker, rpm, nspawn") + raise ModuleFrameworkException("Unsupported MODULE={0}".format( + amodule), "supproted are: docker, rpm, nspawn") def get_correct_profile(): diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index d2df62f..64ba4de 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -37,20 +37,20 @@ from common import * from timeoutlib import Retry -def getBasePackageSet(modulesDict = None, isModule=True, isContainer=False): +def getBasePackageSet(modulesDict=None, isModule=True, isContainer=False): """ Get list of base packages (for bootstrapping of various module types) It is used internally, you should not use it in case you don't know where to use it. - + :param modulesDict: dictionary of dependent modules :param isModule: bool is module :param isContainer: bool is contaner? :return: list of packages to install """ # nspawn container need to install also systemd to be able to boot - out=[] - brmod="base-runtime" - brmod_profiles=["container", "baseimage"] + out = [] + brmod = "base-runtime" + brmod_profiles = ["container", "baseimage"] BASEPACKAGESET_WORKAROUND = ["systemd"] BASEPACKAGESET_WORKAROUND_NOMODULE = ["systemd", "yum"] pdc = None @@ -61,7 +61,8 @@ def getBasePackageSet(modulesDict = None, isModule=True, isContainer=False): pdc.setLatestPDC(brmod, modulesDict[brmod]) for pr in brmod_profiles: if pdc.getmoduleMD()['data']['profiles'].get(pr): - basepackageset = pdc.getmoduleMD()['data']['profiles'][pr]['rpms'] + basepackageset = pdc.getmoduleMD( + )['data']['profiles'][pr]['rpms'] break if isContainer: out = basepackageset @@ -75,12 +76,13 @@ def getBasePackageSet(modulesDict = None, isModule=True, isContainer=False): print_info("ALL packages to install:", out) return out + class PDCParser(): """ Class for parsing PDC data via some setters line setFullVersion, setViaFedMsg, setLatestPDC """ - @Retry(attempts=DEFAULTRETRYCOUNT*5, timeout=DEFAULTRETRYTIMEOUT, delay=20, error=PDCExc("RETRY: Unable to get data from PDC")) + @Retry(attempts=DEFAULTRETRYCOUNT * 5, timeout=DEFAULTRETRYTIMEOUT, delay=20, error=PDCExc("RETRY: Unable to get data from PDC")) def __getDataFromPdc(self): """ Internal method, do not use it @@ -89,8 +91,9 @@ class PDCParser(): """ PDC = "%s/?variant_name=%s&variant_version=%s&variant_release=%s&active=True" % ( PDCURL, self.name, self.stream, self.version) - print_info("Attemt to contact PDC (may take longer time) with query:", PDC) - out=json.load(urllib.urlopen(PDC))["results"] + print_info( + "Attemt to contact PDC (may take longer time) with query:", PDC) + out = json.load(urllib.urlopen(PDC))["results"] if out: self.pdcdata = out[-1] else: @@ -142,7 +145,7 @@ class PDCParser(): :return: str """ - #rpmrepo = "http://kojipkgs.fedoraproject.org/repos/%s/latest/%s" % ( + # rpmrepo = "http://kojipkgs.fedoraproject.org/repos/%s/latest/%s" % ( # self.pdcdata["koji_tag"] + "-build", ARCH) rpmrepo = "https://kojipkgs.stg.fedoraproject.org/compose/branched/jkaluza/latest-Fedora-Modular-26/compose/Server/%s/os/" % ARCH return rpmrepo @@ -208,7 +211,8 @@ class PDCParser(): :return: str """ - utils.process.run("{HOSTPACKAGER} install createrepo koji".format(**trans_dict), ignore_status=True) + utils.process.run("{HOSTPACKAGER} install createrepo koji".format( + **trans_dict), ignore_status=True) dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version) absdir = os.path.abspath(dirname) if os.path.exists(absdir): @@ -221,16 +225,18 @@ class PDCParser(): if len(pkgbouid) > 4: print_debug("DOWNLOADING: %s" % foo) - @Retry(attempts=DEFAULTRETRYCOUNT*10, timeout=DEFAULTRETRYTIMEOUT*60, delay=DEFAULTRETRYTIMEOUT, error=KojiExc("RETRY: Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT*10))) + @Retry(attempts=DEFAULTRETRYCOUNT * 10, timeout=DEFAULTRETRYTIMEOUT * 60, delay=DEFAULTRETRYTIMEOUT, error=KojiExc("RETRY: Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT * 10))) def tmpfunc(): a = utils.process.run( "cd %s; koji download-build %s -a %s -a noarch" % - (absdir, pkgbouid, ARCH), shell=True, verbose=is_debug(),ignore_status=True) + (absdir, pkgbouid, ARCH), shell=True, verbose=is_debug(), ignore_status=True) if a.exit_status == 1: if "packages available for" in a.stdout.strip(): - print_debug('UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) + print_debug( + 'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) else: - raise KojiExc('UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) + raise KojiExc( + 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) tmpfunc() utils.process.run( "cd %s; createrepo -v %s" % diff --git a/moduleframework/setup.py b/moduleframework/setup.py index 9699cb2..bf472bc 100644 --- a/moduleframework/setup.py +++ b/moduleframework/setup.py @@ -54,7 +54,7 @@ class Module(module_framework.CommonFunctions): allmodulerpms = " ".join(self.whattoinstall['rpms']) if self.baseruntimeyaml: allbasertrpms = " ".join(self.baseruntimeyaml['data'][ - 'profiles']['default']['rpms']) + 'profiles']['default']['rpms']) if allbasertrpms is not None and allmodulerpms is not None: utils.process.run( "yumdownloader --destdir=%s --resolve %s %s" % diff --git a/moduleframework/timeoutlib.py b/moduleframework/timeoutlib.py index c4784ff..fc36141 100755 --- a/moduleframework/timeoutlib.py +++ b/moduleframework/timeoutlib.py @@ -27,6 +27,7 @@ from common import print_info log = logging.getLogger('avocado.test') + class Timeout(object): def __init__(self, retry, timeout): self.retry = retry @@ -49,6 +50,7 @@ class Timeout(object): signal.alarm(0) signal.signal(signal.SIGALRM, self.orig_sighand) + class NOPTimeout(object): def __init__(self, *args, **kwargs): pass @@ -59,8 +61,9 @@ class NOPTimeout(object): def __exit__(self, *args, **kwargs): pass + class Retry(object): - def __init__(self, attempts = 1, timeout = None, exceptions = (Exception,), error = None, inverse = False, delay = None): + def __init__(self, attempts=1, timeout=None, exceptions=(Exception,), error=None, inverse=False, delay=None): """ Try to run things ATTEMPTS times, at max, each attempt must not exceed TIMEOUT seconds. Restart only when one of EXCEPTIONS is raised, all other exceptions will just bubble up. @@ -113,8 +116,8 @@ class Retry(object): while True: if delay is not None: - log.debug("Sleeping for delay:", delay) - time.sleep(delay) + log.debug("Sleeping for delay:", delay) + time.sleep(delay) with self.timeout_wrapper(self, self.timeout): start_time = time.time() @@ -146,6 +149,7 @@ class Retry(object): return __wrap + if __name__ == '__main__': class IFailedError(Exception): pass @@ -153,13 +157,13 @@ if __name__ == '__main__': white_horse = [] # Simple "try so many times, and die" case - @Retry(attempts = 5, error = IFailedError('Too many retries!')) - def do_something1(a, b, c, d = 79): + @Retry(attempts=5, error=IFailedError('Too many retries!')) + def do_something1(a, b, c, d=79): white_horse.append(d) raise IFailedError() try: - do_something1(2, 4, 6, d = 97) + do_something1(2, 4, 6, d=97) except IFailedError as e: retry = do_something1.func_closure[1].cell_contents @@ -169,16 +173,16 @@ if __name__ == '__main__': assert retry.timeouts_triggered == 0 except Exception as e: - import sys, traceback + import sys + import traceback print >> sys.stderr, traceback.format_exc() assert False, 'Unexpected exception raised: %s' % repr(e) - # Now with timeout black_horse = [] brown_horse = [] - @Retry(attempts = 2, timeout = 5, error = IFailedError('Too many retries!')) + @Retry(attempts=2, timeout=5, error=IFailedError('Too many retries!')) def do_something2(a, b): black_horse.append(b) time.sleep(30) @@ -196,12 +200,13 @@ if __name__ == '__main__': assert retry.failed_attempts == 2 except Exception as e: - import sys, traceback + import sys + import traceback print >> sys.stderr, traceback.format_exc() assert False, 'Unexpected exception raised: %s' % repr(e) # And react only to a set of exceptions - @Retry(attempts = 3, exceptions = (ValueError,)) + @Retry(attempts=3, exceptions=(ValueError,)) def do_something3(): raise IndexError('This one goes right to the top') @@ -215,12 +220,13 @@ if __name__ == '__main__': assert retry.timeouts_triggered == 0 except Exception as e: - import sys, traceback + import sys + import traceback print >> sys.stderr, traceback.format_exc() assert False, 'Unexpected exception raised: %s' % repr(e) # Use inverted result of wrapped fn - @Retry(attempts = 1 , timeout = 1, error = IFailedError('Too many retries!'), inverse = True) + @Retry(attempts=1, timeout=1, error=IFailedError('Too many retries!'), inverse=True) def do_something4(): raise IFailedError('No, I did not!') @@ -229,7 +235,7 @@ if __name__ == '__main__': # Test delay usage red_horse = [] - @Retry(attempts = 5, timeout = 5, error = IFailedError('Too many retries!'), delay = 20) + @Retry(attempts=5, timeout=5, error=IFailedError('Too many retries!'), delay=20) def do_something5(): red_horse.append(time.time()) time.sleep(10) # should be enough to get killed by watchdog @@ -248,11 +254,14 @@ if __name__ == '__main__': assert retry.timeouts_triggered == 5 for i in range(1, 5): - assert red_horse[i] - red_horse[i - 1] >= 20.0, 'Interval #%i was shorter than expected: %f' % (i, red_horse[i] - red_horse[i - 1]) + assert red_horse[i] - red_horse[i - 1] >= 20.0, 'Interval #%i was shorter than expected: %f' % ( + i, red_horse[i] - red_horse[i - 1]) - assert (end_time - start_time) >= (4 * 20.0 + 5.0), 'All attempts took shorter time than expected: %f' % (end_time - start_time) + assert (end_time - start_time) >= (4 * 20.0 + + 5.0), 'All attempts took shorter time than expected: %f' % (end_time - start_time) except Exception as e: - import sys, traceback + import sys + import traceback print >> sys.stderr, traceback.format_exc() assert False, 'Unexpected exception raised: %s' % repr(e) diff --git a/moduleframework/version.py b/moduleframework/version.py index 544c0b3..ffa1f04 100644 --- a/moduleframework/version.py +++ b/moduleframework/version.py @@ -17,9 +17,11 @@ def version_func(): for line in infile.readlines(): if "Version: " in line: return line[16:].strip() - raise BaseException("Unable to read Version string from specfile:", SPECFILEPATH) + raise BaseException( + "Unable to read Version string from specfile:", SPECFILEPATH) -VERSION=version_func() + +VERSION = version_func() if __name__ == '__main__': print VERSION From 30b089f0882b3630ce7d5104e8879f637ece03da Mon Sep 17 00:00:00 2001 From: Petr Sklenar Date: Jun 05 2017 13:18:01 +0000 Subject: [PATCH 2/2] code clean up, process --- diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 042efa4..a34a0c2 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -28,18 +28,16 @@ main module provides helpers for various module types and AVOCADO(unittest) clas what you should use for your tests (inherited) """ -import os import re import shutil import yaml import json -import time import urllib import glob from avocado import Test -from avocado import utils from avocado.core import exceptions from avocado.utils import service +from avocado.utils import process from compose_info import ComposeParser import pdc_data from common import * @@ -84,8 +82,8 @@ class CommonFunctions(object): Run commands on host :param command: command to exectute - :param kwargs: (avocado utils.process.run) params like: shell, ignore_status, verbose - :return: avocado.utils.process.run + :param kwargs: (avocado process.run) params like: shell, ignore_status, verbose + :return: avocado.process.run """ try: formattedcommand = command.format(**trans_dict) @@ -94,7 +92,7 @@ class CommonFunctions(object): "Command is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ " "or }}, possible values in trans_dict are:", trans_dict) - return utils.process.run("%s" % formattedcommand, **kwargs) + return process.run("%s" % formattedcommand, **kwargs) def installTestDependencies(self, packages=None): """ @@ -153,22 +151,19 @@ class CommonFunctions(object): 'packages'].get('rpms') else [] packages_profiles = [] for x in self.config['packages'].get('profiles') if self.config[ - 'packages'].get('profiles') else []: + 'packages'].get('profiles') else []: packages_profiles = packages_profiles + \ - self.getModulemdYamlconfig()[ - 'data']['profiles'][x]['rpms'] + self.getModulemdYamlconfig()['data']['profiles'][x]['rpms'] out += packages_rpm + packages_profiles elif self.getModulemdYamlconfig()['data'].get('profiles') and self.getModulemdYamlconfig()['data'][ - 'profiles'].get(get_correct_profile()): - out += self.getModulemdYamlconfig( - )['data']['profiles'][get_correct_profile()]['rpms'] + 'profiles'].get(get_correct_profile()): + out += self.getModulemdYamlconfig()['data']['profiles'][get_correct_profile()]['rpms'] else: # fallback solution when it is not known what to install out.append("bash") else: - out += self.getModulemdYamlconfig( - )['data']['profiles'][profile]['rpms'] + out += self.getModulemdYamlconfig()['data']['profiles'][profile]['rpms'] print_info("PCKGs to install inside module:", out) return out @@ -293,8 +288,7 @@ class ContainerHelper(CommonFunctions): :return: None """ if not os.path.isfile('/usr/bin/docker-current'): - self.runHost("{HOSTPACKAGER} install docker", - verbose=is_not_silent()) + self.runHost("{HOSTPACKAGER} install docker", verbose=is_not_silent()) def __prepareContainer(self): """ @@ -303,7 +297,7 @@ class ContainerHelper(CommonFunctions): :return: None """ if self.tarbased is False and self.jmeno == self.icontainer and "docker.io" not in self.info[ - 'container']: + 'container']: registry = re.search("([^/]*)", self.icontainer).groups()[0] if registry not in open('/etc/sysconfig/docker', 'rw').read(): with open("/etc/sysconfig/docker", "a") as myfile: @@ -326,8 +320,7 @@ class ContainerHelper(CommonFunctions): elif "docker=" in self.icontainer: pass else: - self.runHost("docker pull %s" % - self.jmeno, verbose=is_not_silent()) + self.runHost("docker pull %s" % self.jmeno, verbose=is_not_silent()) self.containerInfo = json.loads( self.runHost( @@ -346,8 +339,7 @@ class ContainerHelper(CommonFunctions): if 'start' in self.info and self.info['start']: self.docker_id = self.runHost( "%s -d %s" % - (self.info['start'], - self.jmeno), shell=True, ignore_bg_processes=True, + (self.info['start'], self.jmeno), shell=True, ignore_bg_processes=True, verbose=is_not_silent()).stdout else: self.docker_id = self.runHost( @@ -366,11 +358,9 @@ class ContainerHelper(CommonFunctions): self.getPackageList())), ignore_status=True, verbose=False) if a.exit_status == 0: - print_info( - "Packages installed via {HOSTPACKAGER}", a.stdout) + print_info("Packages installed via {HOSTPACKAGER}", a.stdout) elif b.exit_status == 0: - print_info( - "Packages installed via {GUESTPACKAGER}", b.stdout) + print_info("Packages installed via {GUESTPACKAGER}", b.stdout) else: print_info( "Nothing installed (nor via {HOSTPACKAGER} nor {GUESTPACKAGER}), but package list is not empty", @@ -388,10 +378,8 @@ class ContainerHelper(CommonFunctions): """ if self.status(): try: - self.runHost("docker stop %s" % - self.docker_id, verbose=is_not_silent()) - self.runHost("docker rm %s" % - self.docker_id, verbose=is_not_silent()) + self.runHost("docker stop %s" % self.docker_id, verbose=is_not_silent()) + self.runHost("docker rm %s" % self.docker_id, verbose=is_not_silent()) except Exception as e: print_debug(e, "docker already removed") pass @@ -403,8 +391,8 @@ class ContainerHelper(CommonFunctions): :return: bool """ if self.docker_id and self.docker_id[ - : 12] in self.runHost( - "docker ps", shell=True, verbose=is_not_silent()).stdout: + : 12] in self.runHost( + "docker ps", shell=True, verbose=is_not_silent()).stdout: return True else: return False @@ -415,7 +403,7 @@ class ContainerHelper(CommonFunctions): :param command: str :param kwargs: dict - :return: avocado.utils.process.run + :return: avocado.process.run """ self.start() return self.runHost( @@ -432,8 +420,7 @@ class ContainerHelper(CommonFunctions): :return: None """ self.start() - self.runHost("docker cp %s %s:%s" % - (src, self.docker_id, dest), verbose=is_not_silent()) + self.runHost("docker cp %s %s:%s" % (src, self.docker_id, dest), verbose=is_not_silent()) def copyFrom(self, src, dest): """ @@ -444,8 +431,7 @@ class ContainerHelper(CommonFunctions): :return: None """ self.start() - self.runHost("docker cp %s:%s %s" % - (self.docker_id, src, dest), verbose=is_not_silent()) + self.runHost("docker cp %s:%s %s" % (self.docker_id, src, dest), verbose=is_not_silent()) def __callSetupFromConfig(self): """ @@ -454,8 +440,7 @@ class ContainerHelper(CommonFunctions): :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -464,8 +449,7 @@ class ContainerHelper(CommonFunctions): :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) class RpmHelper(CommonFunctions): @@ -493,9 +477,8 @@ class RpmHelper(CommonFunctions): def setModuleDependencies(self): temprepositories = {} if self.getModulemdYamlconfig()["data"].get("dependencies") and self.getModulemdYamlconfig()["data"][ - "dependencies"].get("requires"): - temprepositories = self.getModulemdYamlconfig( - )["data"]["dependencies"]["requires"] + "dependencies"].get("requires"): + temprepositories = self.getModulemdYamlconfig()["data"]["dependencies"]["requires"] temprepositories_cycle = dict(temprepositories) for x in temprepositories_cycle: pdc = pdc_data.PDCParser() @@ -546,8 +529,7 @@ class RpmHelper(CommonFunctions): else: if not self.repos: for dep in self.moduledeps: - alldrepos.append(get_latest_repo_url( - dep, self.moduledeps[dep])) + alldrepos.append(get_latest_repo_url(dep, self.moduledeps[dep])) if get_correct_url(): self.repos = [get_correct_url()] + alldrepos elif self.info.get('repo'): @@ -562,8 +544,7 @@ class RpmHelper(CommonFunctions): if not self.whattoinstallrpm: self.bootstrappackages = pdc_data.getBasePackageSet(modulesDict=self.moduledeps, isModule=get_if_module(), isContainer=False) - self.whattoinstallrpm = " ".join( - set(self.getPackageList() + self.bootstrappackages)) + self.whattoinstallrpm = " ".join(set(self.getPackageList() + self.bootstrappackages)) def tearDown(self): """ @@ -603,8 +584,7 @@ gpgcheck=0 a = self.runHost( "%s --disablerepo=* --enablerepo=%s* --allowerasing install %s" % - (trans_dict["HOSTPACKAGER"], self.moduleName, - self.whattoinstallrpm), ignore_status=True, + (trans_dict["HOSTPACKAGER"], self.moduleName, self.whattoinstallrpm), ignore_status=True, verbose=is_not_silent()) b = self.runHost( "%s --disablerepo=* --enablerepo=%s* --allowerasing distro-sync" % @@ -626,13 +606,10 @@ gpgcheck=0 """ try: if 'status' in self.info and self.info['status']: - a = self.runHost( - self.info['status'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + a = self.runHost(self.info['status'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) else: - a = self.runHost("%s" % command, shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) - print_debug("command:", a.command, "stdout:", - a.stdout, "stderr:", a.stderr) + a = self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) return True except BaseException: return False @@ -645,11 +622,9 @@ gpgcheck=0 :return: None """ if 'start' in self.info and self.info['start']: - self.runHost(self.info['start'], shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info['start'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) else: - self.runHost("%s" % command, shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def stop(self, command="/bin/true"): """ @@ -659,19 +634,17 @@ gpgcheck=0 :return: None """ if 'stop' in self.info and self.info['stop']: - self.runHost(self.info['stop'], shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info['stop'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) else: - self.runHost("%s" % command, shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def run(self, command="ls /", **kwargs): """ Run command inside module, for RPM based it is same as runHost :param command: str of command to execute - :param kwargs: dict from avocado.utils.process.run - :return: avocado.utils.process.run + :param kwargs: dict from avocado.process.run + :return: avocado.process.run """ return self.runHost('bash -c "%s"' % command.replace('"', r'\"'), **kwargs) @@ -703,8 +676,7 @@ gpgcheck=0 :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -713,8 +685,7 @@ gpgcheck=0 :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) class NspawnHelper(RpmHelper): @@ -732,8 +703,7 @@ class NspawnHelper(RpmHelper): relative change root path """ super(NspawnHelper, self).__init__() - self.__selinuxState = self.runHost( - "getenforce", ignore_status=True).stdout.strip() + self.__selinuxState = None time.time() actualtime = time.time() if get_if_do_cleanup(): @@ -759,8 +729,9 @@ class NspawnHelper(RpmHelper): if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): # TODO: workaround because systemd nspawn is now working well in F-25 # (failing because of selinux) - self.runHost("setenforce Permissive", - ignore_status=True, verbose=is_not_silent()) + self.__selinuxState = self.runHost( + "getenforce", ignore_status=True).stdout.strip() + self.runHost("setenforce Permissive", ignore_status=True, verbose=is_not_silent()) self.setModuleDependencies() self.setRepositoriesAndWhatToInstall() self.installTestDependencies() @@ -771,25 +742,21 @@ class NspawnHelper(RpmHelper): def __is_killed(self): for foo in range(DEFAULTRETRYTIMEOUT): time.sleep(foo) - out = self.runHost("machinectl status %s" % - self.jmeno, verbose=is_debug(), ignore_status=True) + out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) if out.exit_status != 0: print_debug("NSPAWN machine %s stopped" % self.jmeno) return True - raise NspawnExc("Unable to stop machine %s within %d" % - (self.jmeno, DEFAULTRETRYTIMEOUT)) + raise NspawnExc("Unable to stop machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT)) def __is_booted(self): for foo in range(DEFAULTRETRYTIMEOUT): time.sleep(foo) - out = self.runHost("machinectl status %s" % - self.jmeno, verbose=is_debug(), ignore_status=True) + out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) if "logind.service" in out.stdout: time.sleep(2) print_debug("NSPAWN machine %s booted" % self.jmeno) return True - raise NspawnExc("Unable to start machine %s within %d" % - (self.jmeno, DEFAULTRETRYTIMEOUT)) + raise NspawnExc("Unable to start machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT)) def __prepareSetup(self): """ @@ -801,14 +768,12 @@ class NspawnHelper(RpmHelper): shutil.rmtree(self.chrootpath, ignore_errors=True) os.mkdir(self.chrootpath) try: - self.runHost("machinectl terminate %s" % - self.jmeno, verbose=is_debug()) + self.runHost("machinectl terminate %s" % self.jmeno, verbose=is_debug()) self.__is_killed() except BaseException: pass if not os.path.exists(os.path.join(self.chrootpath, "usr")): - self.runHost("{HOSTPACKAGER} install systemd-container", - verbose=is_not_silent()) + self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent()) repos_to_use = "" counter = 0 for repo in self.repos: @@ -858,14 +823,12 @@ gpgcheck=0 try: os.makedirs(os.path.dirname(srcto)) except Exception as e: - print_debug( - e, "Unable to create DIR (already created)", srcto) + print_debug(e, "Unable to create DIR (already created)", srcto) pass try: shutil.copytree(src, srcto) except Exception as e: - print_debug(e, "Unable to copy files from:", - src, "to:", srcto) + print_debug(e, "Unable to copy files from:", src, "to:", srcto) pass pkipath = "/etc/pki/rpm-gpg" pkipath_ch = os.path.join(self.chrootpath, pkipath[1:]) @@ -875,8 +838,7 @@ gpgcheck=0 pass for filename in glob.glob(os.path.join(pkipath, '*')): shutil.copy(filename, pkipath_ch) - print_info("repo prepared for microdnf:", insiderepopath, - open(insiderepopath, 'r').read()) + print_info("repo prepared for microdnf:", insiderepopath, open(insiderepopath, 'r').read()) def __bootMachine(self): @@ -885,7 +847,7 @@ gpgcheck=0 def tempfnc(): print_debug("starting container via command:", "systemd-nspawn --machine=%s -bD %s" % (self.jmeno, self.chrootpath)) - nspawncont = utils.process.SubProcess( + nspawncont = process.SubProcess( "systemd-nspawn --machine=%s -bD %s" % (self.jmeno, self.chrootpath), verbose=is_debug()) nspawncont.start() @@ -906,13 +868,10 @@ gpgcheck=0 """ try: if 'status' in self.info and self.info['status']: - a = self.run(self.info['status'], shell=True, - verbose=False, ignore_bg_processes=True) + a = self.run(self.info['status'], shell=True, verbose=False, ignore_bg_processes=True) else: - a = self.run("%s" % command, shell=True, - verbose=False, ignore_bg_processes=True) - print_debug("command:", a.command, "stdout:", - a.stdout, "stderr:", a.stderr) + a = self.run("%s" % command, shell=True, verbose=False, ignore_bg_processes=True) + print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) return True except BaseException: return False @@ -951,8 +910,8 @@ gpgcheck=0 systemd-run should be used, but in F-25 it does not contain --wait option :param command: str command to be executed - :param kwargs: dict parameters passed to avocado.utils.process.run - :return: avocado.utils.process.run + :param kwargs: dict parameters passed to avocado.process.run + :return: avocado.process.run """ lpath = "/var/tmp" comout = self.runHost( @@ -962,7 +921,7 @@ gpgcheck=0 '"', r'\"'), pin=lpath), - **kwargs) + **kwargs) try: if not kwargs: kwargs = {} @@ -984,14 +943,14 @@ gpgcheck=0 if comout.exit_status == 0 or should_ignore: return comout else: - raise utils.process.CmdError(comout.command, comout) + raise process.CmdError(comout.command, comout) def selfcheck(self): """ Test if default command will pass, it is more important for nspawn, because it happens that it does not returns anything - :return: avocado.utils.process.run + :return: avocado.process.run """ return self.run().stdout @@ -1026,8 +985,7 @@ gpgcheck=0 :return: None """ self.stop() - self.runHost("machinectl poweroff %s" % - self.jmeno, verbose=is_not_silent()) + self.runHost("machinectl poweroff %s" % self.jmeno, verbose=is_not_silent()) # self.nspawncont.stop() self.__is_killed() if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): @@ -1048,8 +1006,7 @@ gpgcheck=0 :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -1058,8 +1015,7 @@ gpgcheck=0 :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, - ignore_bg_processes=True, verbose=is_not_silent()) + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) # INTERFACE CLASS FOR GENERAL TESTS OF MODULES @@ -1143,7 +1099,7 @@ class AvocadoTest(Test): :param args: command :param kwargs: shell, ignore_status, verbose - :return: object avocado.utils.process.run + :return: object avocado.process.run """ return self.backend.run(*args, **kwargs) @@ -1193,7 +1149,7 @@ class AvocadoTest(Test): :param args: pass thru :param kwargs: pass thru - :return: object of avocado.utils.process.run + :return: object of avocado.process.run """ return self.backend.runHost(*args, **kwargs) @@ -1215,8 +1171,7 @@ class AvocadoTest(Test): :return: str """ self.start() - allpackages = self.run( - r'rpm -qa --qf="%{{name}}\n"', verbose=is_not_silent()).stdout.split('\n') + allpackages = self.run(r'rpm -qa --qf="%{{name}}\n"', verbose=is_not_silent()).stdout.split('\n') return allpackages def copyTo(self, *args, **kwargs): @@ -1272,7 +1227,7 @@ class ContainerAvocadoTest(AvocadoTest): :return: bool """ if key in self.backend.containerInfo['Labels'] and ( - value in self.backend.containerInfo['Labels'][key]): + value in self.backend.containerInfo['Labels'][key]): return True return False @@ -1316,7 +1271,7 @@ def get_correct_backend(): readconfig = CommonFunctions() readconfig.loadconfig() if "default_module" in readconfig.config and readconfig.config[ - "default_module"] is not None and amodule is None: + "default_module"] is not None and amodule is None: amodule = readconfig.config["default_module"] if amodule == 'docker': return ContainerHelper(), amodule @@ -1325,8 +1280,7 @@ def get_correct_backend(): elif amodule == 'nspawn': return NspawnHelper(), amodule else: - raise ModuleFrameworkException("Unsupported MODULE={0}".format( - amodule), "supproted are: docker, rpm, nspawn") + raise ModuleFrameworkException("Unsupported MODULE={0}".format(amodule), "supproted are: docker, rpm, nspawn") def get_correct_profile():