From c1e906caf2374d508720f4083109ee187f2b6796 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Nov 02 2017 04:29:31 +0000 Subject: Issue 26: Default to omitting build dependencies Build dependencies for existing packages tend not to be curated particularly well, so self-hosting modulemd drafts are often nigh-unreadable. This changes the default behaviour to be to omit module build dependencies from the SRPM component list entirely, with only identified module level build dependencies being listed. Attempts to make a module self-hosting can be requested by setting `--build-deps` to a suitable number of levels (the level limit ensures dependency cycles don't cause an infinite loop) --- diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index f3ec46d..cf3d0dd 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -33,7 +33,14 @@ class ModtoolsCLI(object): "-o", metavar="FILE", dest="output_fname", - help="Write to FILE instead of stdout" + help="Write to FILE instead of stdout." + ) + parser_rpm2module.add_argument( + "--build-deps", + metavar="N", + default=0, + dest="build_deps_iterations", + help="Attempt to ensure N levels of build dependencies (Default: %(default)s)." ) parser_rpm2module.add_argument( "pkgs", @@ -68,7 +75,7 @@ def run(): logging.basicConfig(level=logging.INFO) if cli.args.cmd_name == 'rpm2module': mg = ModuleGenerator(cli.args.pkgs) - mg.run(cli.args.output_fname) + mg.run(cli.args.output_fname, cli.args.build_deps_iterations) elif cli.args.cmd_name == 'fetch-metadata': _repodata.download_repo_metadata() diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index f63636a..32e7b2d 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -27,9 +27,8 @@ class ModuleGenerator(object): self.pkgs = pkgs self.pkg = None self.mmd = modulemd.ModuleMetadata() - self._calculate_dependencies() - def _calculate_dependencies(self): + def _calculate_dependencies(self, build_deps_iterations): pkgs = self.pkgs _repodata._populate_module_reverse_lookup() pool = _depchase.make_pool("x86_64") @@ -41,13 +40,14 @@ class ModuleGenerator(object): module_build_deps = set() resolved_build_deps = set() build_srpms = set() - for i in range(10): - # Arbitrary bound of 10 levels of SRPM bootstrapping + for i in range(build_deps_iterations+1): + # Give up on making the module self-hosting after the requested + # number of iterations new_module_build_deps, remaining_build_deps = _categorise_deps(build_deps, allow_bootstrap=True) module_build_deps |= new_module_build_deps resolved_build_deps |= (build_deps - remaining_build_deps) build_deps -= resolved_build_deps - if build_deps: + if build_deps and i < build_deps_iterations: new_build_srpms, remaining_build_deps = _depchase.ensure_buildable(pool, build_deps) build_srpms |= new_build_srpms resolved_build_deps |= (build_deps - remaining_build_deps) @@ -55,7 +55,8 @@ class ModuleGenerator(object): if not build_deps: break else: - logging.warn("Failed to close out build dependencies after 10 iteration") + if build_deps_iterations: + logging.warn(f"Failed to close out build dependencies after {build_deps_iterations} iterations") self.module_run_deps = module_run_deps self.module_build_deps = module_build_deps run_srpm_names = {_name_only(n) for n in run_srpms} @@ -149,8 +150,9 @@ class ModuleGenerator(object): else: return 0 - def run(self, output_fname): + def run(self, output_fname, build_deps_iterations): if len(self.pkgs) == 1: self._get_pkg_info() + self._calculate_dependencies(build_deps_iterations) self._update_module_md() self._save_module_md(output_fname) diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index 4da5864..8fd3d0b 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -3,7 +3,8 @@ import os.path from _fedmod.cli import ModtoolsCLI from _fedmod.module_generator import ModuleGenerator -def _generate_modulemd(rpms): +def _generate_modulemd(rpms, build_deps_iterations=0): + # TODO: Actually test the CLI arg parsing via a subprocess cmd_input = list(['rpm2module']) cmd_input.extend(rpms) cli = ModtoolsCLI(cmd_input) @@ -12,7 +13,7 @@ def _generate_modulemd(rpms): output_fname = rpms[0] + '.yaml' else: output_fname = 'modulemd-output.yaml' - mg.run(output_fname) + mg.run(output_fname, build_deps_iterations) return mg, output_fname @@ -34,16 +35,16 @@ class TestSinglePackageInput(object): # Expected description for 'grep' assert modmd.summary == "Pattern matching utilities" - assert (modmd.description == 'The GNU versions of commonly used grep utilities. Grep searches through\n' + - 'textual input for lines which contain a match to a specified pattern and then\n' + - 'prints the matching lines. GNU\'s grep utilities include grep, egrep and fgrep.\n\n' + - 'GNU grep is needed by many scripts, so it shall be installed on every system.') + assert modmd.description == ( + "The GNU versions of commonly used grep utilities. Grep searches through\n" + "textual input for lines which contain a match to a specified pattern and then\n" + "prints the matching lines. GNU\'s grep utilities include grep, egrep and fgrep.\n\n" + "GNU grep is needed by many scripts, so it shall be installed on every system." + ) # Expected licenses for 'grep' - assert len(modmd.module_licenses) == 1 - assert sorted(modmd.module_licenses) == sorted(['MIT']) - assert len(modmd.content_licenses) == 1 - assert sorted(modmd.content_licenses) == sorted(['GPLv3+']) + assert modmd.module_licenses == {'MIT'} + assert modmd.content_licenses == {'GPLv3+'} # Only given modules are listed in the public API assert sorted(modmd.api.rpms) == sorted(self.input_rpms) @@ -53,7 +54,7 @@ class TestSinglePackageInput(object): # Expected module dependencies for grep assert set(modmd.buildrequires) == set() - assert set(modmd.requires) == set(('platform',)) + assert set(modmd.requires) == {'platform',} @@ -78,8 +79,7 @@ class TestMultiplePackageInput(object): assert modmd.description == "" # Expected licenses for grep + haproxy - assert len(modmd.module_licenses) == 1 - assert sorted(modmd.module_licenses) == sorted(['MIT']) + assert modmd.module_licenses == {'MIT'} assert len(modmd.content_licenses) == 0 # This doesn't seem right... # Only given modules are listed in the public API @@ -90,5 +90,185 @@ class TestMultiplePackageInput(object): assert set(modmd.components.rpms) == expected_components # Expected module dependencies for grep + haproxy - assert set(modmd.buildrequires) == set(('platform',)) - assert set(modmd.requires) == set(('platform',)) + assert set(modmd.buildrequires) == {'platform',} + assert set(modmd.requires) == {'platform',} + + +class TestRecursiveBuildDeps(object): + + def setup(self): + self.input_rpms = input_rpms = ('mariadb',) + self.md, self.output_fname = _generate_modulemd(input_rpms, 100) + + def teardown(self): + os.remove(self.output_fname) + + def test_generated_modulemd_file(self): + # File exists with expected name + assert os.path.isfile('mariadb.yaml') + + # Check details of generated module metadata + modmd = self.md.mmd + + # Descriptive metadata + assert modmd.summary == "A community developed branch of MySQL" + assert modmd.description == ( + "MariaDB is a community developed branch of MySQL.\n" + "MariaDB is a multi-user, multi-threaded SQL database server.\n" + "It is a client/server implementation consisting of a server daemon (mysqld)\n" + "and many different client programs and libraries. The base package\n" + "contains the standard MariaDB/MySQL client programs and generic MySQL files." + ) + + # Expected licenses + assert modmd.module_licenses == {'MIT'} + assert modmd.content_licenses == {'GPLv2 with exceptions and LGPLv2 and BSD'} + + # Only given modules are listed in the public API + assert sorted(modmd.api.rpms) == sorted(self.input_rpms) + + # MariaDB's complicated test suite poses some real challenges for + # build dependency resolution, even when the generator is kind of + # cheating and relying on the actual Fedora MariaDB module at runtime + + # Expected components + expected_components = set(self.input_rpms) + expected_components |= { + 'systemtap', + 'multilib-rpm-config', + 'libselinux', + 'libuv', + 'jsoncpp', + 'python-nose', + 'numpy', + 'openblas', + 'Judy', + 'setools', + 'rhash', + 'pyparsing', + 'boost', + 'libsemanage', + 'jemalloc', + } + assert set(modmd.components.rpms) == expected_components + + # Expected module dependencies + expected_build_deps = { + 'platform', + 'mariadb', + 'python2', + 'networking-base', + 'host', + 'python3', + 'perl', + 'installer', + 'mariadb', + } + assert set(modmd.buildrequires) == expected_build_deps + assert set(modmd.requires) == {'platform', 'perl', 'mariadb'} + + +class TestNonRecursiveBuildDeps(object): + + def setup(self): + self.input_rpms = input_rpms = ('graphite-web',) + self.md, self.output_fname = _generate_modulemd(input_rpms) + + def teardown(self): + os.remove(self.output_fname) + + def test_generated_modulemd_file(self): + # File exists with expected name + assert os.path.isfile('graphite-web.yaml') + + # Check details of generated module metadata + modmd = self.md.mmd + + # Descriptive metadata + assert modmd.summary == "A Django web application for enterprise scalable realtime graphing" + assert modmd.description == ( + "Graphite consists of a storage backend and a web-based visualization frontend.\n" + "Client applications send streams of numeric time-series data to the Graphite\n" + "backend (called carbon), where it gets stored in fixed-size database files\n" + "similar in design to RRD. The web frontend provides user interfaces\n" + "for visualizing this data in graphs as well as a simple URL-based API for\n" + "direct graph generation.\n\n" + "Graphite's design is focused on providing simple interfaces (both to users and\n" + "applications), real-time visualization, high-availability, and enterprise\n" + "scalability." + ) + + # Expected licenses + assert modmd.module_licenses == {'MIT'} + assert modmd.content_licenses == {'ASL 2.0'} + + # Only given modules are listed in the public API + assert sorted(modmd.api.rpms) == sorted(self.input_rpms) + + # Expected components + expected_components = set(self.input_rpms) + expected_components |= { + 'python-twisted', + 'python-simplejson', + 'python-service-identity', + 'python-crypto', + 'python-xpyb', + 'python-memcached', + 'python-zope-interface', + 'python-whitenoise', + 'python-whisper', + 'pyparsing', + 'dejavu-fonts', + 'python-zope-event', + 'python-django', + 'python-django-tagging', + 'python-attrs', + 'python-carbon', + 'python-fadvise', + 'pyserial', + } + assert set(modmd.components.rpms) == expected_components + + # Expected module dependencies + # graphite-web is currently generating runtime module dependencies + # than expected, so it makes for an interesting test case to look for + # cases where fedmod is picking up module implementation details + + expected_build_requires = { + 'platform', + 'fonts', + 'python2', + 'python2-ecosystem' + } + _unexpected_build_requires = { + 'pki', + 'samba', + 'python3', + 'python3-ecosystem', + 'installer', + 'X11-base', + 'freeipa', + 'cloud-init', + 'perl', + 'fonts', + 'freeipa', + 'java', + 'host', + 'networking-base', + } + assert set(modmd.buildrequires) == expected_build_requires | _unexpected_build_requires + expected_requires = { + 'platform', + 'httpd', + 'fonts', + 'python2', + } + _unexpected_requires = { + 'pki', + 'samba', + 'python3-ecosystem', + 'installer', + 'X11-base', + 'freeipa', + } + assert set(modmd.requires) == expected_requires | _unexpected_requires