From f001e766db033524c28a07d3390b5809412ef1ab Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Nov 10 2017 22:27:28 +0000 Subject: [PATCH 1/2] resolve-deps: Add a --json flag to get full output in JSON form When --json is passed to 'fedmod resolve-deps', return output in JSON form that contains, for each package The NVR of the package The NVR of the corresponding source package The package in the transaction that satisfied each requirement This is intended to allow other tools to provide higher-level reporting or analysis while sharing the underlying data sources and dependency-reporting logic of fedmod. --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 9f2d340..588df58 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -1,3 +1,4 @@ +import collections import configparser import itertools import logging @@ -69,39 +70,52 @@ def get_sourcepkg(p, s=None, only_name=False): assert len(solvables) == 1 return solvables[0] -def print_transaction(pool, transaction): +def _get_dependency_details(pool, transaction): candq = transaction.newpackages() - if log.getEffectiveLevel() <= logging.INFO: - tb = smartcols.Table() - tb.title = "DEPENDENCY INFORMATION" - cl = tb.new_column("INFO") - cl.tree = True - cl_match = tb.new_column("MATCH") - for p in candq: - ln = tb.new_line() - ln[cl] = str(p) - for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES): - lns = tb.new_line(ln) - lns[cl] = str(dep) - matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep)) - if not matches and str(dep).startswith("/"): - # Append provides by files - # TODO: use Dataiterator for getting filelist - matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) - # It was possible to resolve set, so something is wrong here - assert matches - first = True - for m in matches: - if first: - lnc = lns - else: - lnss = tb.new_line(lns) - lnc = lnss - first = False - lnc[cl_match] = str(m) - log.info(tb) - -def _solve(solver, pkgnames): + result = {} + for p in candq: + pkg_details = {} + for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES): + matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep)) + if not matches and str(dep).startswith("/"): + # Append provides by files + # TODO: use Dataiterator for getting filelist + matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) + # It was possible to resolve set, so something is wrong here + assert matches + pkg_details[str(dep)] = sorted(str(m) for m in matches) + result[str(p)] = pkg_details + + return result + +def print_transaction(details): + tb = smartcols.Table() + tb.title = "DEPENDENCY INFORMATION" + cl = tb.new_column("INFO") + cl.tree = True + cl_match = tb.new_column("MATCH") + for p in sorted(details): + ln = tb.new_line() + ln[cl] = p + deps = details[p] + for dep in sorted(deps): + matches = deps[dep] + lns = tb.new_line(ln) + lns[cl] = dep + first = True + for m in matches: + if first: + lnc = lns + else: + lnss = tb.new_line(lns) + lnc = lnss + first = False + lnc[cl_match] = m + log.info(tb) + +FullInfo = collections.namedtuple('FullInfo', ['name', 'rpm', 'srpm', 'requires']) + +def _solve(solver, pkgnames, full_info=False): """Given a set of package names, returns a list of solvables to install""" pool = solver.pool @@ -125,19 +139,31 @@ def _solve(solver, pkgnames): for problem in problems: log.warn(problem) - print_transaction(pool, solver.transaction()) - result = set() + if log.getEffectiveLevel() <= logging.INFO or full_info: + dep_details = _get_dependency_details(pool, solver.transaction()) + if log.getEffectiveLevel() <= logging.INFO: + print_transaction(dep_details) + + if full_info: + result = [] + else: + result = set() for s in solver.transaction().newpackages(): if s.name.startswith("fedora-release"): # Relying on the F27 metadata injects irrelevant fedora-release deps continue if s.arch in ("src", "nosrc"): continue - # Ensure the solvables don't outlive the solver that created them - result.add(s.name) + # Ensure the solvables don't outlive the solver that created them by + # extracting the information we want but not returning the solvable. + if full_info: + rpm = str(s) + result.append(FullInfo(s.name, rpm, s.lookup_sourcepkg()[:-4], dep_details[rpm])) + else: + result.add(s.name) return result -def ensure_buildable(pool, pkgnames): +def ensure_buildable(pool, pkgnames, full_info=False): """Given a set of solvables, returns a set of source packages & build deps""" # The given package set may not be installable on its own # That's OK, since other modules will provide those packages @@ -152,7 +178,8 @@ def make_pool(arch): _DEFAULT_HINTS = ("glibc-minimal-langpack",) -def ensure_installable(pool, pkgnames, hints=_DEFAULT_HINTS, recommendations=False): +def ensure_installable(pool, pkgnames, hints=_DEFAULT_HINTS, + recommendations=False, full_info=False): """Iterate over the resolved dependency set for the given packages *hints*: Packages that have higher priority when more than one package @@ -174,7 +201,7 @@ def ensure_installable(pool, pkgnames, hints=_DEFAULT_HINTS, recommendations=Fal # Ignore weak deps solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1) - return _solve(solver, pkgnames) + return _solve(solver, pkgnames, full_info=full_info) def print_reldeps(pool, pkg): sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index 59f3cea..05e5a79 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -88,6 +88,11 @@ class ModtoolsCLI(object): help="Module to be used as a dependency. Can be used multiple times.", ) parser_resolve_deps.add_argument( + "--json", + action='store_true', + help="Output dependencies in JSON format with extra information.", + ) + parser_resolve_deps.add_argument( "pkgs", metavar='PKGS', nargs='+', @@ -163,7 +168,9 @@ def run(): ) elif cli.args.cmd_name == 'resolve-deps': rq = ModuleRepoquery() - rq.list_pkg_deps(cli.args.pkgs, cli.args.module_dependency) + rq.list_pkg_deps(cli.args.pkgs, + module_deps=cli.args.module_dependency, + json_output=cli.args.json) elif cli.args.cmd_name == 'module-packages': rq = ModuleRepoquery() rq.list_rpms_in_module(cli.args.module, full_nevra=cli.args.full_nevra) diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index ddaa3a4..6d7edf8 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import json import sys import modulemd import logging @@ -36,7 +37,7 @@ class ModuleRepoquery(object): else: print(_name_only(name)) - def list_pkg_deps(self, pkgs, module_deps): + def list_pkg_deps(self, pkgs, module_deps=None, json_output=False): _repodata._populate_module_reverse_lookup() pkgs_in_modules = set() if module_deps: @@ -45,11 +46,24 @@ class ModuleRepoquery(object): pkgs_in_modules |= set(map(lambda x: _name_only(x), rpm_names)) pool = _depchase.make_pool("x86_64") - run_deps = _depchase.ensure_installable(pool, pkgs) - rpm_names = run_deps - pkgs_in_modules - if rpm_names: - for name in rpm_names: - print(name) + if json_output: + run_deps = _depchase.ensure_installable(pool, pkgs, full_info=True) + result = [] + for info in run_deps: + if info.name in pkgs_in_modules: + continue + result.append({ + 'rpm': info.rpm, + 'srpm': info.srpm, + 'requires': info.requires, + }) + json.dump(result, sys.stdout, indent=4, sort_keys=True) + else: + run_deps = _depchase.ensure_installable(pool, pkgs) + rpm_names = run_deps - pkgs_in_modules + if rpm_names: + for name in rpm_names: + print(name) def list_modularized_pkgs(self, duplicate_only=False, list_modules=False): _repodata._populate_module_reverse_lookup() From ff2d2e55ad3b0905089b85b9faf84d82c55f36d9 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Nov 10 2017 22:27:28 +0000 Subject: [PATCH 2/2] For --verbose and --json, pick a single package for each dependency Handling the case where a dependency is provided by multiple packages in the transaction adds complexity for a very rare case. Just pick one package arbitrarily in this case - this makes 'requires' in the JSON output a single RPM name and not an array. --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 588df58..3d8dfa3 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -83,7 +83,9 @@ def _get_dependency_details(pool, transaction): matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) # It was possible to resolve set, so something is wrong here assert matches - pkg_details[str(dep)] = sorted(str(m) for m in matches) + # While multiple packages providing the same thing is certainly possible, it is rare, and + # the confusion from picking one at random is worth the the simplification. + pkg_details[str(dep)] = sorted(str(m) for m in matches)[0] result[str(p)] = pkg_details return result @@ -99,18 +101,9 @@ def print_transaction(details): ln[cl] = p deps = details[p] for dep in sorted(deps): - matches = deps[dep] lns = tb.new_line(ln) lns[cl] = dep - first = True - for m in matches: - if first: - lnc = lns - else: - lnss = tb.new_line(lns) - lnc = lnss - first = False - lnc[cl_match] = m + lns[cl_match] = deps[dep] log.info(tb) FullInfo = collections.namedtuple('FullInfo', ['name', 'rpm', 'srpm', 'requires'])