From bbd7d219f401054e5b461ea9d8b3ccd586c8fc91 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jun 27 2017 12:04:46 +0000 Subject: [PATCH 1/5] initial commit of recursive downloading of all dependent modules to one repository --- diff --git a/moduleframework/common.py b/moduleframework/common.py index ecfc00b..cf867a1 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -153,3 +153,6 @@ def print_debug(*args): """ if is_debug(): print_info(*args) + +def if_recursive_download(): + return bool(os.environ.get("MTF_RECURSIVE_DOWNLOAD")) \ No newline at end of file diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index 4dc3091..e246d32 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -206,6 +206,29 @@ class PDCParser(): out = {} return out + def download_tagged(self,dirname): + for foo in utils.process.run("koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"): + pkgbouid = foo.strip().split(" ")[0] + if len(pkgbouid) > 4: + print_debug("DOWNLOADING: %s" % foo) + + @Retry(attempts=DEFAULTRETRYCOUNT * 10, timeout=DEFAULTRETRYTIMEOUT * 60, delay=DEFAULTRETRYTIMEOUT, + error=KojiExc( + "RETRY: Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT * 10))) + def tmpfunc(): + a = utils.process.run( + "cd %s; koji download-build %s -a %s -a noarch" % + (dirname, pkgbouid, ARCH), shell=True, verbose=is_debug(), ignore_status=True) + if a.exit_status == 1: + if "packages available for" in a.stdout.strip(): + print_debug( + 'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) + else: + raise KojiExc( + 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) + + tmpfunc() + def createLocalRepoFromKoji(self): """ Return string of generated repository located LOCALLY @@ -221,25 +244,9 @@ class PDCParser(): pass else: os.mkdir(absdir) - for foo in utils.process.run( - "koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"): - pkgbouid = foo.strip().split(" ")[0] - if len(pkgbouid) > 4: - print_debug("DOWNLOADING: %s" % foo) - - @Retry(attempts=DEFAULTRETRYCOUNT * 10, timeout=DEFAULTRETRYTIMEOUT * 60, delay=DEFAULTRETRYTIMEOUT, error=KojiExc("RETRY: Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT * 10))) - def tmpfunc(): - a = utils.process.run( - "cd %s; koji download-build %s -a %s -a noarch" % - (absdir, pkgbouid, ARCH), shell=True, verbose=is_debug(), ignore_status=True) - if a.exit_status == 1: - if "packages available for" in a.stdout.strip(): - print_debug( - 'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) - else: - raise KojiExc( - 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) - tmpfunc() + self.download_tagged(absdir) + if if_recursive_download(): + self.generateDepModules() utils.process.run( "cd %s; createrepo -v %s" % (absdir, absdir), shell=True, verbose=is_debug()) From b4783e4193d049302662d2a52a589baff2ae65b2 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jun 27 2017 12:20:44 +0000 Subject: [PATCH 2/5] solved issues with multiple directories, create just one repo, and do not overwrite it --- diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index e246d32..782921c 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -207,6 +207,7 @@ class PDCParser(): return out def download_tagged(self,dirname): + print_info("DOWLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version)) for foo in utils.process.run("koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"): pkgbouid = foo.strip().split(" ")[0] if len(pkgbouid) > 4: @@ -228,6 +229,7 @@ class PDCParser(): 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) tmpfunc() + print_info("DOWLOADING finished" % (self.name, self.stream, self.version)) def createLocalRepoFromKoji(self): """ @@ -238,7 +240,10 @@ class PDCParser(): """ utils.process.run("{HOSTPACKAGER} install createrepo koji".format( **trans_dict), ignore_status=True) - dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version) + if if_recursive_download(): + dirname="localrepo_recursive" + else: + dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version) absdir = os.path.abspath(dirname) if os.path.exists(absdir): pass @@ -246,7 +251,12 @@ class PDCParser(): os.mkdir(absdir) self.download_tagged(absdir) if if_recursive_download(): - self.generateDepModules() + allmodules = self.generateDepModules() + for mo in allmodules: + localrepo = PDCParser() + localrepo.setLatestPDC(mo, allmodules[mo]) + localrepo.download_tagged(dirname) + utils.process.run( "cd %s; createrepo -v %s" % (absdir, absdir), shell=True, verbose=is_debug()) From 964a66b26795d34a60b249cfb193059e18e88f76 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jun 27 2017 12:34:32 +0000 Subject: [PATCH 3/5] added new option to run-them script --- diff --git a/tools/run-them.sh b/tools/run-them.sh index 4e64c25..61f473c 100755 --- a/tools/run-them.sh +++ b/tools/run-them.sh @@ -52,6 +52,7 @@ mkdir -p $AVDIR export XUFILE="$AVDIR/out.xunit" export AVOCADOCMD="avocado run --xunit $XUFILE" export RESULTTOOLS=0 +export MTF_RECURSIVE_DOWNLOAD=yes function getparams_int(){ ADDIT="$1" @@ -122,7 +123,7 @@ function run_modulelint(){ set -x inst_env RESULTTOOLS=$(($RESULTTOOLS+$?)) -PARAMS="`getparams_int`" +PARAMS="`DEBUG=yes getparams_int`" RESULTTOOLS=$(($RESULTTOOLS+$?)) export PARAMS From 540a1397e3293f5c6d4617fd375b0c62d81d0ffb Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jun 27 2017 12:39:20 +0000 Subject: [PATCH 4/5] if is bad name prefix, is_ is better --- diff --git a/moduleframework/common.py b/moduleframework/common.py index cf867a1..1fbda74 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -154,5 +154,5 @@ def print_debug(*args): if is_debug(): print_info(*args) -def if_recursive_download(): +def is_recursive_download(): return bool(os.environ.get("MTF_RECURSIVE_DOWNLOAD")) \ No newline at end of file diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index 782921c..f6cb66c 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -240,7 +240,7 @@ class PDCParser(): """ utils.process.run("{HOSTPACKAGER} install createrepo koji".format( **trans_dict), ignore_status=True) - if if_recursive_download(): + if is_recursive_download(): dirname="localrepo_recursive" else: dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version) @@ -250,7 +250,7 @@ class PDCParser(): else: os.mkdir(absdir) self.download_tagged(absdir) - if if_recursive_download(): + if is_recursive_download(): allmodules = self.generateDepModules() for mo in allmodules: localrepo = PDCParser() From b841e8e2ae86001e51dcd7db042fb6c7113bb1d7 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jun 27 2017 12:44:50 +0000 Subject: [PATCH 5/5] if is bad name prefix, is_ is better --- diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index f6cb66c..5aef6d9 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -241,7 +241,7 @@ class PDCParser(): utils.process.run("{HOSTPACKAGER} install createrepo koji".format( **trans_dict), ignore_status=True) if is_recursive_download(): - dirname="localrepo_recursive" + dirname = "localrepo_recursive" else: dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version) absdir = os.path.abspath(dirname)