From cd8826b6434bfe534e4e7cc6c27646d6504e2a5f Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Apr 11 2018 13:01:17 +0000 Subject: Use libmodulemd instead of modulemd Use the libmodulemd library via gobject-introspection. libmodulemd has support for version 2 of the modulemd specification, and produces better formatted output with human-readable ordering. --- diff --git a/README.md b/README.md index db00f02..ed56a90 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Python installation. Some dependencies aren't currently available from PyPI, and hence need to be installed system-wide: - $ sudo dnf install python3-solv + $ sudo dnf install libmodulemd python3-gobject-base python3-solv ### Additional development dependencies diff --git a/fedmod.spec b/fedmod.spec index d120606..b87c929 100644 --- a/fedmod.spec +++ b/fedmod.spec @@ -13,8 +13,10 @@ BuildArch: noarch BuildRequires: python3-devel BuildRequires: python3-setuptools +Requires: libmodulemd >= 1.2.0 Requires: python3-attrs Requires: python3-click +Requires: python3-gobject-base Requires: python3-lxml Requires: python3-modulemd Requires: python3-PyYAML diff --git a/src/_fedmod/__init__.py b/src/_fedmod/__init__.py index e69de29..f23693c 100644 --- a/src/_fedmod/__init__.py +++ b/src/_fedmod/__init__.py @@ -0,0 +1,3 @@ +import gi + +gi.require_version('Modulemd', '1.0') diff --git a/src/_fedmod/_fetchrepodata.py b/src/_fedmod/_fetchrepodata.py index 2fbce41..78fd4e4 100644 --- a/src/_fedmod/_fetchrepodata.py +++ b/src/_fedmod/_fetchrepodata.py @@ -6,9 +6,10 @@ import os from collections import defaultdict import click -import modulemd import requests +from gi.repository import Modulemd + from attr import attributes, attrib from lxml import etree from requests_toolbelt.downloadutils.tee import tee_to_file @@ -155,19 +156,24 @@ def _write_lookup_caches(): repomd_xml = etree.parse(repomd_fname) repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) - with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: + with gzip.open(repo_modulemd_fname, "rt") as modules_yaml_gz: modules_yaml = modules_yaml_gz.read() - modules = modulemd.loads_all(modules_yaml) module_forward_lookup = {} srpm_reverse_lookup = defaultdict(list) rpm_reverse_lookup = defaultdict(list) - for module in modules: - module_forward_lookup[module.name] = list(set(module.artifacts.rpms)) - for srpmname in module.components.rpms: - srpm_reverse_lookup[srpmname].append(module.name) - for rpmname in module.artifacts.rpms: + + objects = Modulemd.objects_from_string(modules_yaml) + for module in (o for o in objects if isinstance(o, Modulemd.Module)): + module_name = module.props.name + artifacts = module.props.rpm_artifacts.get() + + module_forward_lookup[module_name] = list(set(artifacts)) + components = module.props.components_rpm + for srpmname in components: + srpm_reverse_lookup[srpmname].append(module_name) + for rpmname in artifacts: rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] - rpm_reverse_lookup[rpmprefix].append(module.name) + rpm_reverse_lookup[rpmprefix].append(module_name) # Cache the lookup tables as local JSON files print("Caching lookup tables") _write_cache("_MODULE_FORWARD_LOOKUP_CACHE", module_forward_lookup) diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index 950733b..c66ad23 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -5,7 +5,6 @@ import os.path import requests import click import logging -import modulemd import solv import json from ._fetchrepodata import load_cached_repodata diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index 054ceb0..a088304 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -1,6 +1,8 @@ -import sys import modulemd import logging + +from gi.repository import Modulemd + from . import _depchase, _repodata def _name_only(rpm_name): @@ -23,7 +25,7 @@ class ModuleGenerator(object): def __init__(self, pkgs): self.pkgs = pkgs self.core_srpm = None - self.mmd = modulemd.ModuleMetadata() + self.mmd = Modulemd.Module(mdversion=2) self._pool = _depchase.make_pool("x86_64") def _calculate_dependencies(self): @@ -63,27 +65,36 @@ class ModuleGenerator(object): are information taken from SPEC file. :return: """ - self.mmd.add_module_license("MIT") + self.mmd.props.module_licenses.add("MIT") if self.core_srpm is not None: - self.mmd.summary = f"Generated module for {self.core_srpm}" + self.mmd.props.summary = f"Generated module for {self.core_srpm}" else: - self.mmd.summary = f"Generated module for {self.pkgs}" + self.mmd.props.summary = f"Generated module for {self.pkgs}" - self.mmd.description = "Module auto-generated by fedmod" + self.mmd.props.description = "Module auto-generated by fedmod" # Declare the public API for pkg in self.api_srpms: - self.mmd.api.add_rpm(pkg) - self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg)) + self.mmd.props.rpm_api.add(pkg) + + component = Modulemd.ComponentRpm(name=pkg, + rationale="Package in api", + buildorder=self._get_build_order(pkg)) + self.mmd.add_rpm_component(component) # Declare module level dependencies + dependencies = Modulemd.Dependencies() for modname in self.module_run_deps: - self.mmd.buildrequires[modname] = "f28" - self.mmd.requires[modname] = "f28" + dependencies.add_buildrequires(modname, []) + dependencies.add_requires(modname, []) + self.mmd.add_dependencies(dependencies) for pkg in self.run_srpms: - self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg)) + component = Modulemd.ComponentRpm(name=pkg, + rationale="Runtime dependencies", + buildorder=self._get_build_order(pkg)) + self.mmd.add_rpm_component(component) # TODO: Always set content licenses appropriately @@ -92,7 +103,7 @@ class ModuleGenerator(object): def _get_build_order(self, pkg): - if pkg in self.mmd.api.rpms: + if self.mmd.props.rpm_api.contains(pkg): return 10 else: return 0 diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index 0f60f63..83bda27 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -1,6 +1,5 @@ import json import sys -import modulemd import logging import dnf from . import _depchase, _repodata diff --git a/src/setup.py b/src/setup.py index 696a566..664ac92 100644 --- a/src/setup.py +++ b/src/setup.py @@ -24,7 +24,6 @@ setup( ] }, install_requires=[ - 'modulemd', 'click', 'requests', 'requests-toolbelt', diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index f5b9d3a..cbe3003 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -3,7 +3,8 @@ import pytest from click.testing import CliRunner -import modulemd +from gi.repository import Modulemd + from _fedmod.cli import _cli_commands from _fedmod.module_generator import ModuleGenerator @@ -13,8 +14,7 @@ def _generate_modulemd(rpms): runner = CliRunner() result = runner.invoke(_cli_commands, cmd) assert result.exit_code == 0 - modmd = modulemd.ModuleMetadata() - modmd.loads(result.output) + modmd = Modulemd.Module.new_from_string(result.output) return modmd @@ -25,22 +25,30 @@ class TestSinglePackageInput(object): modmd = _generate_modulemd(input_rpms) # Expected description for 'grep' - assert modmd.summary == "Generated module for grep" - assert modmd.description == "Module auto-generated by fedmod" + assert modmd.props.summary == "Generated module for grep" + assert modmd.props.description == "Module auto-generated by fedmod" # Expected licenses for 'grep' - assert modmd.module_licenses == {'MIT'} - assert modmd.content_licenses == set() + assert modmd.props.module_licenses.get() == ['MIT'] + assert modmd.props.content_licenses.get() == [] # Only given modules are listed in the public API - assert sorted(modmd.api.rpms) == sorted(input_rpms) + assert sorted(modmd.props.rpm_api.get()) == sorted(input_rpms) # Expected components for grep - assert set(modmd.components.rpms) == set(input_rpms) + assert set(modmd.props.components_rpm) == set(input_rpms) # Expected module dependencies for grep - assert set(modmd.buildrequires) == {'platform',} - assert set(modmd.requires) == {'platform',} + dependencies = modmd.props.dependencies + assert len(dependencies) == 1 + + buildrequires = dependencies[0].props.buildrequires + assert set(buildrequires) == {'platform',} + assert buildrequires['platform'].get() == [] + + requires = dependencies[0].props.requires + assert set(requires) == {'platform',} + assert requires['platform'].get() == [] class TestMultiplePackageInput(object): @@ -50,22 +58,28 @@ class TestMultiplePackageInput(object): modmd = _generate_modulemd(input_rpms) # Can't generate descriptive metadata when given multiple RPMs - assert modmd.summary == "Generated module for ('grep', 'haproxy')" - assert modmd.description == "Module auto-generated by fedmod" + assert modmd.props.summary == "Generated module for ('grep', 'haproxy')" + assert modmd.props.description == "Module auto-generated by fedmod" # Expected licenses for grep + haproxy - assert modmd.module_licenses == {'MIT'} - assert len(modmd.content_licenses) == 0 # This doesn't seem right... + assert modmd.props.module_licenses.get() == ['MIT'] + assert modmd.props.content_licenses.get() == [] # This doesn't seem right... # Only given modules are listed in the public API - assert sorted(modmd.api.rpms) == sorted(input_rpms) + assert sorted(modmd.props.rpm_api.get()) == sorted(input_rpms) # Expected components for grep + haproxy expected_components = set(input_rpms) - assert set(modmd.components.rpms) == expected_components + assert set(modmd.props.components_rpm) == expected_components - # Expected module dependencies for grep + haproxy - assert set(modmd.buildrequires) == {'platform',} - assert set(modmd.requires) == {'platform',} + # Expected module dependencies for grep + dependencies = modmd.props.dependencies + assert len(dependencies) == 1 + buildrequires = dependencies[0].props.buildrequires + assert set(buildrequires) == {'platform',} + assert buildrequires['platform'].get() == [] + requires = dependencies[0].props.requires + assert set(requires) == {'platform',} + assert requires['platform'].get() == []