From af2512aeeedc491e852db2c23905f708021edb60 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Dec 04 2017 07:09:04 +0000 Subject: [PATCH 1/5] resolve-deps: Add a --json flag to get full output in JSON form When --json is passed to 'fedmod resolve-deps', return output in JSON form that contains, for each package The NVR of the package The NVR of the corresponding source package The package in the transaction that satisfied each requirement This is intended to allow other tools to provide higher-level reporting or analysis while sharing the underlying data sources and dependency-reporting logic of fedmod. --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 065855e..2e07951 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -1,5 +1,6 @@ """"_depchase: Resolve dependency & build relationships between RPMs and SRPMs""" +import collections import configparser import itertools import logging @@ -71,39 +72,45 @@ def get_sourcepkg(p, s=None, only_name=False): assert len(solvables) == 1 return solvables[0] -def print_transaction(pool, transaction): +def _get_dependency_details(pool, transaction): candq = transaction.newpackages() - if log.getEffectiveLevel() <= logging.INFO: - tb = smartcols.Table() - tb.title = "DEPENDENCY INFORMATION" - cl = tb.new_column("INFO") - cl.tree = True - cl_match = tb.new_column("MATCH") - for p in candq: - ln = tb.new_line() - ln[cl] = str(p) - for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES): - lns = tb.new_line(ln) - lns[cl] = str(dep) - matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep)) - if not matches and str(dep).startswith("/"): - # Append provides by files - # TODO: use Dataiterator for getting filelist - matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) - # It was possible to resolve set, so something is wrong here - assert matches - first = True - for m in matches: - if first: - lnc = lns - else: - lnss = tb.new_line(lns) - lnc = lnss - first = False - lnc[cl_match] = str(m) - log.info(tb) - -def _solve(solver, pkgnames): + result = {} + for p in candq: + pkg_details = {} + for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES): + matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep)) + if not matches and str(dep).startswith("/"): + # Append provides by files + # TODO: use Dataiterator for getting filelist + matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) + # It was possible to resolve set, so something is wrong here + assert matches + # While multiple packages providing the same thing is certainly possible, it is rare, and + # the confusion from picking one at random is worth the the simplification. + pkg_details[str(dep)] = sorted(str(m) for m in matches)[0] + result[str(p)] = pkg_details + + return result + +def print_transaction(details): + tb = smartcols.Table() + tb.title = "DEPENDENCY INFORMATION" + cl = tb.new_column("INFO") + cl.tree = True + cl_match = tb.new_column("MATCH") + for p in sorted(details): + ln = tb.new_line() + ln[cl] = p + deps = details[p] + for dep in sorted(deps): + lns = tb.new_line(ln) + lns[cl] = dep + lns[cl_match] = deps[dep] + log.info(tb) + +FullInfo = collections.namedtuple('FullInfo', ['name', 'rpm', 'srpm', 'requires']) + +def _solve(solver, pkgnames, full_info=False): """Given a set of package names, returns a list of solvables to install""" pool = solver.pool @@ -127,19 +134,31 @@ def _solve(solver, pkgnames): for problem in problems: log.warn(problem) - print_transaction(pool, solver.transaction()) - result = set() + if log.getEffectiveLevel() <= logging.INFO or full_info: + dep_details = _get_dependency_details(pool, solver.transaction()) + if log.getEffectiveLevel() <= logging.INFO: + print_transaction(dep_details) + + if full_info: + result = [] + else: + result = set() for s in solver.transaction().newpackages(): if s.name.startswith("fedora-release"): # Relying on the F27 metadata injects irrelevant fedora-release deps continue if s.arch in ("src", "nosrc"): continue - # Ensure the solvables don't outlive the solver that created them - result.add(s.name) + # Ensure the solvables don't outlive the solver that created them by + # extracting the information we want but not returning the solvable. + if full_info: + rpm = str(s) + result.append(FullInfo(s.name, rpm, s.lookup_sourcepkg()[:-4], dep_details[rpm])) + else: + result.add(s.name) return result -def ensure_buildable(pool, pkgnames): +def ensure_buildable(pool, pkgnames, full_info=False): """Given a set of solvables, returns a set of source packages & build deps""" # The given package set may not be installable on its own # That's OK, since other modules will provide those packages @@ -154,7 +173,8 @@ def make_pool(arch): _DEFAULT_HINTS = ("glibc-minimal-langpack",) -def ensure_installable(pool, pkgnames, hints=_DEFAULT_HINTS, recommendations=False): +def ensure_installable(pool, pkgnames, hints=_DEFAULT_HINTS, + recommendations=False, full_info=False): """Iterate over the resolved dependency set for the given packages *hints*: Packages that have higher priority when more than one package @@ -176,7 +196,7 @@ def ensure_installable(pool, pkgnames, hints=_DEFAULT_HINTS, recommendations=Fal # Ignore weak deps solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1) - return _solve(solver, pkgnames) + return _solve(solver, pkgnames, full_info=full_info) def print_reldeps(pool, pkg): sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index 1eeedd3..a6d3be5 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -61,11 +61,13 @@ def rpm2module(pkgs, output, build_deps): @_cli_commands.command('resolve-deps') @click.option("--module-dependency", "-m", multiple=True, metavar="MODULE", help="Module to be used as a dependency. Can be given multiple times.") +@click.option("--json", is_flag=True, default=False, + help="Output dependencies in JSON format with extra information.") @click.argument("pkgs", metavar='PKGS', nargs=-1, required=True) -def resolve_deps(pkgs, module_dependency): +def resolve_deps(pkgs, module_dependency, json): """Report dependencies of the given RPM (or list of RPMs)""" rq = ModuleRepoquery() - rq.list_pkg_deps(pkgs, module_dependency) + rq.list_pkg_deps(pkgs, module_dependency, json) # Listing modules diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index 713b6d0..48a7979 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import json import sys import modulemd import logging @@ -33,7 +34,7 @@ class ModuleRepoquery(object): else: print(_name_only(name)) - def list_pkg_deps(self, pkgs, module_deps): + def list_pkg_deps(self, pkgs, module_deps, json_output): pkgs_in_modules = set() if module_deps: for module in module_deps: @@ -41,11 +42,24 @@ class ModuleRepoquery(object): pkgs_in_modules |= set(map(lambda x: _name_only(x), rpm_names)) pool = _depchase.make_pool("x86_64") - run_deps = _depchase.ensure_installable(pool, pkgs) - rpm_names = run_deps - pkgs_in_modules - if rpm_names: - for name in sorted(rpm_names): - print(name) + if json_output: + run_deps = _depchase.ensure_installable(pool, pkgs, full_info=True) + result = [] + for info in run_deps: + if info.name in pkgs_in_modules: + continue + result.append({ + 'rpm': info.rpm, + 'srpm': info.srpm, + 'requires': info.requires, + }) + json.dump(result, sys.stdout, indent=4, sort_keys=True) + else: + run_deps = _depchase.ensure_installable(pool, pkgs) + rpm_names = run_deps - pkgs_in_modules + if rpm_names: + for name in rpm_names: + print(name) def list_modularized_pkgs(self, duplicate_only=False, list_modules=False): rpm_names = _repodata.get_rpm_reverse_lookup() From 190390e8c06cad2fc100a4a0c342cfe01ef4a32f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Dec 04 2017 07:18:05 +0000 Subject: [PATCH 2/5] Report lists to handle ambiguous deps --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 2e07951..6665cb1 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -85,9 +85,11 @@ def _get_dependency_details(pool, transaction): matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) # It was possible to resolve set, so something is wrong here assert matches - # While multiple packages providing the same thing is certainly possible, it is rare, and - # the confusion from picking one at random is worth the the simplification. - pkg_details[str(dep)] = sorted(str(m) for m in matches)[0] + # While multiple packages providing the same thing is rare, it's + # the kind of duplication we want fedmod to be able to help find. + # So we always return a list here, even though it will normally + # only have one entry in it + pkg_details[str(dep)] = sorted(str(m) for m in matches) result[str(p)] = pkg_details return result @@ -103,9 +105,18 @@ def print_transaction(details): ln[cl] = p deps = details[p] for dep in sorted(deps): + matches = deps[dep] lns = tb.new_line(ln) lns[cl] = dep - lns[cl_match] = deps[dep] + first = True + for m in matches: + if first: + lnc = lns + else: + lnss = tb.new_line(lns) + lnc = lnss + first = False + lnc[cl_match] = m log.info(tb) FullInfo = collections.namedtuple('FullInfo', ['name', 'rpm', 'srpm', 'requires']) From 9f52e233600e668f2d1d207c9b453366da85c76b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Dec 04 2017 07:25:49 +0000 Subject: [PATCH 3/5] Keep function signature compatible --- diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index 48a7979..637c833 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -34,7 +34,7 @@ class ModuleRepoquery(object): else: print(_name_only(name)) - def list_pkg_deps(self, pkgs, module_deps, json_output): + def list_pkg_deps(self, pkgs, module_deps, json_output=False): pkgs_in_modules = set() if module_deps: for module in module_deps: From 2a5f7a22f5cacf92135067f67359ff746445f3ea Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Dec 05 2017 05:10:24 +0000 Subject: [PATCH 4/5] Add a basic 'resolve-deps --json' test case --- diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index 637c833..30ca443 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -54,6 +54,7 @@ class ModuleRepoquery(object): 'requires': info.requires, }) json.dump(result, sys.stdout, indent=4, sort_keys=True) + print() else: run_deps = _depchase.ensure_installable(pool, pkgs) rpm_names = run_deps - pkgs_in_modules diff --git a/tests/test_cli_ux.py b/tests/test_cli_ux.py index ce1710d..e9c45b7 100644 --- a/tests/test_cli_ux.py +++ b/tests/test_cli_ux.py @@ -1,6 +1,7 @@ """Tests for the general behaviour of the CLI""" import pytest +import json import os import re import sys @@ -55,3 +56,19 @@ class TestHelpMessages(object): def test_subcommand_help(self): for subcommand in EXPECTED_SUBCOMMANDS: assert _run_fedmod([subcommand, "--help"]).returncode == 0 + +class TestJSONFormatting(object): + # Test that commands that emit JSON, emit well-formed JSON + # TODO: extract the JSON schema definition & validation code from + # fabric8-analytics, and re-use it in fedmod + + def test_resolve_deps_json(self): + result = _run_fedmod(["resolve-deps", "--json", "graphite-web"]) + assert result.returncode == 0 + assert result.stdout.endswith(b"\n") + deps = json.loads(result.stdout) + for dep in deps: + assert dep.keys() == set(["rpm", "srpm", "requires"]) + for requirement, provider in dep["requires"].items(): + assert isinstance(requirement, str) + assert isinstance(provider, list) From fd33260422e91552ba08229cfdb82a2e146abc7e Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Dec 05 2017 05:16:02 +0000 Subject: [PATCH 5/5] Add docs for 'resolve-deps --json' --- diff --git a/src/README.md b/src/README.md index c8f842e..5e3ff51 100644 --- a/src/README.md +++ b/src/README.md @@ -66,6 +66,43 @@ $ fedmod resolve-deps -m host -m platform pkg pkg3 ``` +Passing the `--json` option both requests additional information about +dependencies (giving the SRPM name and runtime requirements for each RPM), and +changes the output format to be a structured JSON list rather than just a +simple list of names: + +``` +$ fedmod resolve-deps --json setup +[ + { + "requires": { + "system-release": [ + "fedora-release-27-1.noarch" + ] + }, + "rpm": "setup-2.10.10-1.fc27.noarch", + "srpm": "setup-2.10.10-1.fc27.src" + }, + { + "requires": { + "fedora-gpg-keys = 27-1": [ + "fedora-gpg-keys-27-1.noarch" + ], + "system-release(27)": [ + "fedora-release-27-1.noarch" + ] + }, + "rpm": "fedora-repos-27-1.noarch", + "srpm": "fedora-repos-27-1.src" + }, + { + "requires": {}, + "rpm": "fedora-gpg-keys-27-1.noarch", + "srpm": "fedora-repos-27-1.src" + } +] +``` + **Find package in modules** Finds out whether a certain package has been modularized and in which module(s).