From dfcdbbaff2e07bbd53263adac34fec2e16fdd766 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 11 2016 21:00:22 +0000 Subject: [PATCH 1/4] don't ignore source data in cg_import --- diff --git a/hub/kojihub.py b/hub/kojihub.py index fb6f9cb..d8e85a6 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4805,7 +4805,7 @@ class CG_Importer(object): raise koji.GenericError("Build already exists: %r" % buildinfo) else: # gather needed data - buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) + buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra', 'source']) # epoch is not in the metadata spec, but we allow it to be specified buildinfo['epoch'] = metadata['build'].get('epoch', None) buildinfo['start_time'] = \ From b0da550dfc3c8fc1f2e0fb6056d64bbd26b32017 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 11 2016 21:00:22 +0000 Subject: [PATCH 2/4] ignore tagless buildroots in BuildTagTest --- diff --git a/hub/kojihub.py b/hub/kojihub.py index d8e85a6..ba7a972 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -7571,6 +7571,9 @@ class BuildTagTest(koji.policy.BaseSimpleTest): if br_id is None: continue tagname = get_buildroot(br_id)['tag_name'] + if tagname is None: + # content generator buildroots might not have tag info + continue for pattern in args: if fnmatch.fnmatch(tagname, pattern): return True From c796c2534c8f7b84049359f18fd691fc7afce6d7 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: May 12 2016 11:22:58 +0000 Subject: [PATCH 3/4] add -top-broken parameter to switch to a output with a list of most blocking build failures --- diff --git a/util/koji-shadow b/util/koji-shadow index 7b9da74..3ccec65 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -62,7 +62,6 @@ def _(args): """Stub function for translation""" return args - class SubOption(object): """A simple container to help with tracking ConfigParser data""" pass @@ -140,6 +139,8 @@ def get_options(): help=_("Rules: list of package names to never replace")) parser.add_option("--tag-build", action="store_true", default=False, help=_("tag sucessful builds into the tag we are building, default is to not tag")) + parser.add_option("--top-broken", action="store_true", default=False, + help=_("switch to different output which shows failed builds and how many other packages they block")) parser.add_option("--arches", help=_("arches to use when creating tags")) parser.add_option("--priority", type="int", default=5, @@ -843,7 +844,10 @@ class BuildTracker(object): self.checkJobs(tag) self.rebuildMissing() if len(self.builds) % 50 == 0: - self.report() + if options.top_broken: + self.report_topbroken() + else: + self.report() return build def scanTag(self, tag): @@ -1105,6 +1109,81 @@ class BuildTracker(object): task_id = session.build(src, None, opts={'repo_id': repo_id}, priority=options.priority) return task_id + def report_topbroken(self): + brokenpackage = {} + thispackageblocks = {} + self.report_brief() + for state in ('broken', 'noroot', 'blocked'): + builds = self.state_idx[state].values() + not_replaced = [b for b in builds if not b.substitute] + n_replaced = len(builds) - len(not_replaced) + for b in not_replaced: + if not b.nvr in brokenpackage: + brokenpackage.update({b.nvr:[b.nvr]}) + else: + brokenpackage[b.nvr].append(b.nvr) + for build in self.state_idx['brokendeps'].values(): + for dep_id in build.deps: + dep = self.builds.get(dep_id) + if not dep: + #unscanned + #possible because we short circuit the earlier scan on problems + #we don't really know if this one is a problem or not, so just + #skip it. + continue + if dep.state in ('common', 'pending', 'missing'): + #not a problem + continue + nvr = dep.nvr + if dep.substitute: + dep2 = self.getSubstitute(dep.substitute) + if dep2: + #we have a substitution, so not a problem + continue + #otherwise the substitution is the problem + nvr = dep.substitute + if dep.state == 'broken': + brokenpackage[dep.nvr].append(build.nvr) + elif dep.state == 'brokendeps': + found = 0; + for i in brokenpackage: + if dep.nvr in brokenpackage[i]: + brokenpackage[i] += [build.nvr] + found = 1 + if found == 0: + if not dep.nvr in thispackageblocks: + thispackageblocks.update({dep.nvr:[build.nvr]}) + else: + if len(thispackageblocks[dep.nvr]) > 100: + thispackageblocks[dep.nvr].append('') + else: + thispackageblocks[dep.nvr].append(build.nvr) + # group all brokendeps + for package in thispackageblocks: + for blockedpackage in thispackageblocks: + if package in thispackageblocks[blockedpackage]: + thispackageblocks[blockedpackage] += thispackageblocks[package] + #print(thispackageblocks: %s" % thispackageblocks + for package in thispackageblocks: + for blockedpackage in brokenpackage: + if package in brokenpackage[blockedpackage]: + if len(brokenpackage[blockedpackage]) > 500: + continue + else: + brokenpackage[blockedpackage] += thispackageblocks[package] + for package in brokenpackage: +# Remove duplicates, use 'set' as order isn't important: + brokenpackage[package] = set(brokenpackage[package]) +# enable for debugging only, otherwise log files can get huge: +# print "brokenpackage: %s %s" % (package, brokenpackage[package]) + order = [(len(c), nvr) for (nvr, c) in brokenpackage.iteritems()] + if order: + order.sort() + order.reverse() + print "-- top broken packages --" + for (c, nvr) in order[:20]: + print "%s (%i)" % (nvr, c) + def report(self): print "-- %s --" % time.asctime() self.report_brief() @@ -1296,7 +1375,10 @@ def main(args): else: print "Working on tag %s" % (tag) tracker.scanTag(tag) - tracker.report() + if options.top_broken: + tracker.report_topbroken() + else: + tracker.report() tracker.runRebuilds(tag)