From 3ea1489b9c4a7bfdd0933c98bec70eb5bfd18a92 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Dec 07 2018 12:10:19 +0000 Subject: [PATCH 1/5] summarizer: ignore cached metadata when given a yaml Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/cli.py b/_fedmod/cli.py index a70e90a..f76c060 100644 --- a/_fedmod/cli.py +++ b/_fedmod/cli.py @@ -196,8 +196,9 @@ def lint(modulemd, min_level): @click.argument("modules", metavar='MODULES', nargs=-1, required=False) @click.option("--add-file", "-f", metavar="FILE", multiple=True, type=click.Path(exists=True), - help="Additional modulemd files to check." - " Can be given multiple times.") + help="Modulemd files to check. " + "Causes cached metadata to be ignored. " + "Can be given multiple times.") @click.option("--tree", "-t", is_flag=True, default=False, help="Print output as a tree") def summarize(modules, add_file, tree): diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index cdad351..0332b5c 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -25,6 +25,7 @@ gi.require_version('Modulemd', '1.0') # noqa: E402 from gi.repository import Modulemd import smartcols +from collections import defaultdict from . import _repodata @@ -81,38 +82,35 @@ def _print_summary(profiles, sdefaults, pdefaults, deps, restrict_to, as_tree): print('\nHint: [d]efault') -def _add_module_metadata(yaml_files, profiles, dstreams, dprofiles, deps): - for yaml in yaml_files: - assert yaml.endswith('.yaml'), "Not a yaml file: {}".format(yaml) - - mmd_index, failures = Modulemd.index_from_file(yaml) - assert len(failures) == 0, failures - - for module_name, index in mmd_index.items(): - for module in index.get_streams().values(): - # local modulemd files might miss some info like context or - # version. So let's make sure nsvc is consistent - ctxt = module.get_context() or '' - nsvc = f'{module_name}:{module.get_stream()}:' \ - f'{module.get_version()}:{ctxt}' - plist = list(module.get_profiles().keys()) - profiles[nsvc] = sorted(set(profiles.get(nsvc, []) + plist)) - - for dep in module.get_dependencies(): - deplist = set(deps.get(nsvc, [])) - for m, s in dep.peek_requires().items(): - deplist.add(f"{m}:{','.join(s.get())}" - if len(s.get()) else m) - deps[nsvc] = sorted(deplist) - - defaults = index.get_defaults() - if not defaults: - continue +def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps): + mmd_index, failures = Modulemd.index_from_file(yaml) + assert len(failures) == 0, failures + + for module_name, index in mmd_index.items(): + for module in index.get_streams().values(): + # local modulemd files might miss some info like context or + # version. So let's make sure nsvc is consistent + ctxt = module.get_context() or '' + nsvc = f'{module_name}:{module.get_stream()}:' \ + f'{module.get_version()}:{ctxt}' + plist = list(module.get_profiles().keys()) + profiles[nsvc] = sorted(set(profiles.get(nsvc, []) + plist)) + + for dep in module.get_dependencies(): + deplist = set(deps.get(nsvc, [])) + for m, s in dep.peek_requires().items(): + deplist.add(f"{m}:{','.join(s.get())}" + if len(s.get()) else m) + deps[nsvc] = sorted(deplist) + + defaults = index.get_defaults() + if not defaults: + continue - # Local module metadata can overwrite metadata from repo - dstreams[module_name] = defaults.peek_default_stream() - for s, pset in defaults.peek_profile_defaults().items(): - dprofiles[module_name][s] = pset.get() + # Local module metadata can overwrite metadata from repo + dstreams[module_name] = defaults.peek_default_stream() + for s, pset in defaults.peek_profile_defaults().items(): + dprofiles[module_name][s] = pset.get() def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False): @@ -122,16 +120,20 @@ def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False): *restrict_list*: if present, restricts output to modules supplied - *yaml_files*: additional yaml files to parse and include in the summary + *yaml_files*: yaml files to parse and include in the summary. If any, + locally cached metadata will be ignored """ - profiles = _repodata.get_modules_profiles_lookup().copy() - dstreams = _repodata.get_modules_default_streams_lookup().copy() - dprofiles = _repodata.get_modules_default_profiles_lookup().copy() - deps = _repodata.get_modules_dependencies_lookup().copy() - + profiles, dstreams, dprofiles, deps = {}, {}, defaultdict(dict), {} if yaml_files: - _add_module_metadata(yaml_files, profiles, dstreams, dprofiles, deps) + for yaml in yaml_files: + assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}" + _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps) + else: + profiles = _repodata.get_modules_profiles_lookup() + dstreams = _repodata.get_modules_default_streams_lookup() + dprofiles = _repodata.get_modules_default_profiles_lookup() + deps = _repodata.get_modules_dependencies_lookup() restrict_list = restrict_list or [] _print_summary(profiles, dstreams, dprofiles, deps, restrict_list, as_tree) diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py index f3df520..4cd2a6b 100644 --- a/tests/test_module_summary.py +++ b/tests/test_module_summary.py @@ -54,8 +54,6 @@ class TestModuleSummary(object): summarize_modules(yaml_files=[spec_v2_yaml_path]) out, err = capfd.readouterr() - assert self.matches('testmodule', 'master', '20180405123256', - 'c2c572ec', 'default', 'platform:f29', out) assert self.matches('foo', 'stream-name', '20160927144203', 'c0ffee43', 'buildroot, container, default, minimal, ' + 'srpm-buildroot', 'compatible:v3,v4,extras,' + From 629d25f6106a98899b9cf90a77b55a0d28991a08 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Dec 07 2018 12:12:14 +0000 Subject: [PATCH 2/5] summarizer: add glob pattern matching for filtering This can be helpful if one wants to check all modules with a stream called 'default': just pass '*:default:*' to summarize-module and the ouptut will be restrict to those modules. It's more helpful than just filtering by module name which can be easily done with grep. Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index 0332b5c..7f9c208 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -25,12 +25,13 @@ gi.require_version('Modulemd', '1.0') # noqa: E402 from gi.repository import Modulemd import smartcols +from fnmatch import fnmatch from collections import defaultdict from . import _repodata -def _print_summary(profiles, sdefaults, pdefaults, deps, restrict_to, as_tree): +def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree): tb = smartcols.Table() cl = tb.new_column('Name') cl.tree = as_tree @@ -41,11 +42,11 @@ def _print_summary(profiles, sdefaults, pdefaults, deps, restrict_to, as_tree): cl_deps = tb.new_column('Dependencies') parent_ln = {} for nsvc, plist in sorted(profiles.items()): - modname, sname, version, context = nsvc.split(':') - - if restrict_to and modname not in restrict_to: + if mfilter and not any((fnmatch(nsvc, p) for p in mfilter)): continue + modname, sname, version, context = nsvc.split(':') + def is_def_strm(s): return s == sdefaults.get(modname, '') @@ -113,12 +114,13 @@ def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps): dprofiles[module_name][s] = pset.get() -def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False): +def summarize_modules(mfilter=None, yaml_files=None, as_tree=False): """ Load Modulemd objects from each repository in repo_list and print a summary of the modules found with a summary of their streams and profiles. - *restrict_list*: if present, restricts output to modules supplied + *mfilter*: if present, restricts output to the nsvc supplied. Glob pattern + can be used, e.g for filtering by stream: '*:master:*' *yaml_files*: yaml files to parse and include in the summary. If any, locally cached metadata will be ignored @@ -135,5 +137,5 @@ def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False): dprofiles = _repodata.get_modules_default_profiles_lookup() deps = _repodata.get_modules_dependencies_lookup() - restrict_list = restrict_list or [] - _print_summary(profiles, dstreams, dprofiles, deps, restrict_list, as_tree) + mfilter = mfilter or [] + _print_summary(profiles, dstreams, dprofiles, deps, mfilter, as_tree) diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py index 4cd2a6b..1a5628d 100644 --- a/tests/test_module_summary.py +++ b/tests/test_module_summary.py @@ -35,7 +35,7 @@ class TestModuleSummary(object): 'c2c572ec', 'default', 'platform:f29', out) def test_summarize_modules_restricted(self, capfd): - summarize_modules(['reviewboard', 'django']) + summarize_modules(['reviewboard*', 'django*']) out, err = capfd.readouterr() assert self.matches('reviewboard', '2.5', '20180828143308', '083bce86', From 8a38f68414d2834290107b3f95e0001e2828a6c9 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Dec 07 2018 12:13:12 +0000 Subject: [PATCH 3/5] summarizer: update method documentation Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index 7f9c208..aa5d351 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -114,21 +114,25 @@ def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps): dprofiles[module_name][s] = pset.get() -def summarize_modules(mfilter=None, yaml_files=None, as_tree=False): +def summarize_modules(mfilter=None, yamls=None, as_tree=False): """ - Load Modulemd objects from each repository in repo_list and print a summary - of the modules found with a summary of their streams and profiles. + Load Modulemd objects from each yaml file in the `yamls` list and print a + summary of the modules found with streams, context, version, profiles and + dependencies information. + If no files are passed, local cached metadata is used instead. *mfilter*: if present, restricts output to the nsvc supplied. Glob pattern can be used, e.g for filtering by stream: '*:master:*' - *yaml_files*: yaml files to parse and include in the summary. If any, - locally cached metadata will be ignored + *yamls*: list of yaml files to parse. If any, locally cached metadata will + be ignored + + *as_tree*: print the summary in tree format, grouping information. """ profiles, dstreams, dprofiles, deps = {}, {}, defaultdict(dict), {} - if yaml_files: - for yaml in yaml_files: + if yamls: + for yaml in yamls: assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}" _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps) else: diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py index 1a5628d..470b786 100644 --- a/tests/test_module_summary.py +++ b/tests/test_module_summary.py @@ -51,7 +51,7 @@ class TestModuleSummary(object): 'c2c572ec', 'default', 'platform:f29', out) def test_summarize_modules_local_files(self, capfd): - summarize_modules(yaml_files=[spec_v2_yaml_path]) + summarize_modules(yamls=[spec_v2_yaml_path]) out, err = capfd.readouterr() assert self.matches('foo', 'stream-name', '20160927144203', 'c0ffee43', From 2030d2be1d36509b33b3ad3f60cdc0295b01ae05 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Dec 07 2018 12:13:18 +0000 Subject: [PATCH 4/5] summarizer: read metadata from url in cmdline It's useful to have a summary of modules from a repository given its url without having to cache it first. Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/cli.py b/_fedmod/cli.py index f76c060..aa76245 100644 --- a/_fedmod/cli.py +++ b/_fedmod/cli.py @@ -199,8 +199,12 @@ def lint(modulemd, min_level): help="Modulemd files to check. " "Causes cached metadata to be ignored. " "Can be given multiple times.") +@click.option("--add-url", "-u", metavar="URL", multiple=True, + help="Repositories to read module metadata from." + "Causes cached metadata to be ignored. " + "Can be given multiple times.") @click.option("--tree", "-t", is_flag=True, default=False, help="Print output as a tree") -def summarize(modules, add_file, tree): +def summarize(modules, add_file, add_url, tree): """Prints a summary of available modules""" - summarize_modules(modules, add_file, tree) + summarize_modules(modules, add_file, add_url, tree) diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index aa5d351..e7f7e0f 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -24,11 +24,12 @@ import gi gi.require_version('Modulemd', '1.0') # noqa: E402 from gi.repository import Modulemd +import tempfile import smartcols from fnmatch import fnmatch from collections import defaultdict -from . import _repodata +from . import _repodata, _fetchrepodata def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree): @@ -83,11 +84,9 @@ def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree): print('\nHint: [d]efault') -def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps): - mmd_index, failures = Modulemd.index_from_file(yaml) - assert len(failures) == 0, failures - - for module_name, index in mmd_index.items(): +def _parse_mmd(mmd_index, profiles, dstreams, dprofiles, deps): + for index in mmd_index: + module_name = index.get_name() for module in index.get_streams().values(): # local modulemd files might miss some info like context or # version. So let's make sure nsvc is consistent @@ -114,12 +113,12 @@ def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps): dprofiles[module_name][s] = pset.get() -def summarize_modules(mfilter=None, yamls=None, as_tree=False): +def summarize_modules(mfilter=None, yamls=None, urls=None, as_tree=False): """ - Load Modulemd objects from each yaml file in the `yamls` list and print a - summary of the modules found with streams, context, version, profiles and - dependencies information. - If no files are passed, local cached metadata is used instead. + Load Modulemd objects from each yaml file in the `yamls` list or each url + in `urls` list and print a summary of the modules found with streams, + context, version, profiles and dependencies information. + If no files and no ulrs are passed, local cached metadata is used instead. *mfilter*: if present, restricts output to the nsvc supplied. Glob pattern can be used, e.g for filtering by stream: '*:master:*' @@ -127,6 +126,9 @@ def summarize_modules(mfilter=None, yamls=None, as_tree=False): *yamls*: list of yaml files to parse. If any, locally cached metadata will be ignored + *urls*: list of repositories to read from. If any, locally cached metadata + will be ignored. + *as_tree*: print the summary in tree format, grouping information. """ @@ -134,7 +136,16 @@ def summarize_modules(mfilter=None, yamls=None, as_tree=False): if yamls: for yaml in yamls: assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}" - _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps) + index, failures = Modulemd.index_from_file(yaml) + assert len(failures) == 0, failures + _parse_mmd(index.values(), profiles, dstreams, dprofiles, deps) + elif urls: + for url in urls: + with tempfile.TemporaryDirectory() as local_path: + rp = _fetchrepodata.RepoPaths(url, local_path) + _fetchrepodata._download_metadata_files(rp) + indexes = _fetchrepodata._read_modules(rp) + _parse_mmd(indexes, profiles, dstreams, dprofiles, deps) else: profiles = _repodata.get_modules_profiles_lookup() dstreams = _repodata.get_modules_default_streams_lookup() From a1fb7c2e5bbe0c68a70fd48f8a72ada27dd1bdcc Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Dec 07 2018 12:25:20 +0000 Subject: [PATCH 5/5] summarizer: replace assert by exception Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index e7f7e0f..4731d42 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -27,6 +27,7 @@ from gi.repository import Modulemd import tempfile import smartcols from fnmatch import fnmatch +from click import ClickException from collections import defaultdict from . import _repodata, _fetchrepodata @@ -135,9 +136,10 @@ def summarize_modules(mfilter=None, yamls=None, urls=None, as_tree=False): profiles, dstreams, dprofiles, deps = {}, {}, defaultdict(dict), {} if yamls: for yaml in yamls: - assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}" index, failures = Modulemd.index_from_file(yaml) - assert len(failures) == 0, failures + if len(failures) != 0: + msgs = "\n".join((str(f.get_gerror()) for f in failures)) + raise ClickException(f"Could not read {yaml}: {msgs}") _parse_mmd(index.values(), profiles, dstreams, dprofiles, deps) elif urls: for url in urls: