From 017cba941435ce6cec24418ed6f9bfb36dd20c49 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Oct 26 2017 11:28:41 +0000 Subject: Assorted code structure related cleanups - make the Python API private (fedmod -> _fedmod) - actually generate the script wrapper correctly - move the shippable parts inside a `src` directory - only list the runtime dependencies in `src/setup.py` - remove some unnecessary dev dependencies --- diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index b52089f..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include MANIFEST.in -include README.md -include LICENSE -include modtools -recursive-include modularity * diff --git a/Pipfile b/Pipfile index cc24c95..4fc5f46 100644 --- a/Pipfile +++ b/Pipfile @@ -1,16 +1,16 @@ [[source]] + url = "https://pypi.python.org/simple" verify_ssl = true + [dev-packages] pytest = "*" six = "*" -snakeviz = "*" + [packages] -fedmod = {editable = true, path = "."} -"requests-toolbelt" = "*" -click = "*" -attrs = "*" \ No newline at end of file +# Runtime dependencies should go in src/setup.py rather than here +fedmod = {editable = true, path = "src"} diff --git a/Pipfile.lock b/Pipfile.lock index f2bc15d..53f29f9 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "eb22a789500751e490310d5c8e96e2d6d51d9f89adcf0f5d88ea79641e1fa8b3" + "sha256": "65629a979a8af096bf2e21c52c0b0ad16b9f7a80006bbd74e10bec82ab03fcda" }, "host-environment-markers": { "implementation_name": "cpython", @@ -26,137 +26,9 @@ ] }, "default": { - "attrs": { - "hashes": [ - "sha256:a7e0d9183f6457de12df7ba6a81f6569c7d6b25f67ad509b5ad52e8545970a2f", - "sha256:5d4d1b99f94d69338f485984127e4473b3ab9e20f43821b0e546cc3b2302fd11" - ], - "version": "==17.2.0" - }, - "certifi": { - "hashes": [ - "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704", - "sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5" - ], - "version": "==2017.7.27.1" - }, - "chardet": { - "hashes": [ - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", - "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" - ], - "version": "==6.7" - }, "fedmod": { "editable": true, - "path": "." - }, - "idna": { - "hashes": [ - "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", - "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" - ], - "version": "==2.6" - }, - "lxml": { - "hashes": [ - "sha256:7a8715539adb41c78129983ba69d852e0102a3f51d559eeb91dce1f6290c4ad0", - "sha256:d3a98dda9831a37ef7f55c5e69c0d276c278f24978f5b36b9fad7eac05a22bfc", - "sha256:1deacd52638da2d7fcb864c3949f0285638ec10e6aace93ce15c6a2e0ed91b95", - "sha256:1548247ea3b50014a3ea55ad9446108df191b6a6e51aa8f5953c95b663f382ff", - "sha256:5da6f5b31ea2b573cb20e88aefc6b49d849d07588ba60871342cae42f569b0d7", - "sha256:12e348eb57fb79ccf91a49b7b937c49a5bbe1d73ba75589674b76a56d064bda0", - "sha256:b9e1735918fc1e83c522b9f1048e6bc5af38af958e4efc843046e4b0075a021b", - "sha256:fafeb4b190bd63ba2bcee2496d99cb7345fafbace6b999403010abdff8c05b72", - "sha256:9007da6fb1b96fb1c9d7bd65e97bbbad60295abc19833d7e67e05314c1868f58", - "sha256:cfbf0b956f33cda3af2a1438a2541549b69a7a240e71de7d8ca819b8f1547aac", - "sha256:0a103253a94cdad86028d273aaebb8b30c75fdf009c23e52cdc8ce88429fd326", - "sha256:b3d5a0ecf0c2c31c404246b6706b2e477159ee07b73be5102389ab250dd67701", - "sha256:e92af0fd08c7d2176ad4be4a7c47fd800d6ee05046b41e36ed579c01fb106c25", - "sha256:feb2144c2ae4035ad57165dd22bdc93b1389158a985c0497a096d39e2b2cd67b", - "sha256:a4433655219b84a360dbdf2c34d9625c3988a272e6fc028222d528ad5902f6a2", - "sha256:db98287cb1488eb103930a64444542f6ffe83694ef392f801aa56d648d905663", - "sha256:307d325ee143b60b9c82912e96e9f4345200c33c8ae00b04b001e4c85fb5f146", - "sha256:5caec9b174dbf927034d588669c62d2a9d0ce447365b20a3463f4daab1e4f03b", - "sha256:fb816595494ce21191764572215f56edfbc6d9fbebd1491c8466502892989689", - "sha256:10399bececdb67f0d9251ecf2dda2abf6ddeee6096741754356f1a3715c8c830", - "sha256:c263fd15d27f3be93485fcd83a495cbbc35352512d9e31644d49a54504a1be2a", - "sha256:c10ad53216d5af2b3ba63e65db793cb7dd7e598e17826938045e32f38b0e4814", - "sha256:d42a5182d4b0953d02e5f46c9f0dc304be736fbaa1c0d2f11326182b9684b5f4", - "sha256:dd7c22bf890d266e72c5e5c8c44555ffbfe4ca2a329da785e7d8b1972fc3ff74", - "sha256:93df9805146980e83834ea9320baa6a56d8aea45f63d7d3cc721f71eb1a1bac6", - "sha256:7ba1b62fe9414d73d493241011df952b72074808debc3a2d6d8a64fb9944edf6", - "sha256:d2c121f5f77bed1e1eddeee23ee76fee8a3d48fa7a3aab589d12942f87778a9e", - "sha256:be3aaeb5f468a49f523f16736ccff7d82af2b4b303292ba3d052b5b28f3fbe47" - ], - "version": "==4.1.0" - }, - "modulemd": { - "hashes": [ - "sha256:59e66590f4a2e301c629cc83f3696723b1da19ee93a038311dc97bf5713dcccd" - ], - "version": "==1.3.2" - }, - "python-dateutil": { - "hashes": [ - "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c", - "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca" - ], - "version": "==2.6.1" - }, - "pyyaml": { - "hashes": [ - "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f", - "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736", - "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269", - "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8", - "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4", - "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1", - "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab", - "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3", - "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8", - "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6", - "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca", - "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8", - "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608", - "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7" - ], - "version": "==3.12" - }, - "requests": { - "hashes": [ - "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", - "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" - ], - "version": "==2.18.4" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" - ], - "version": "==0.8.0" - }, - "six": { - "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" - ], - "version": "==1.11.0" - }, - "urllib3": { - "hashes": [ - "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", - "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" - ], - "version": "==1.22" + "path": "src" } }, "develop": { @@ -180,23 +52,6 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" ], "version": "==1.11.0" - }, - "snakeviz": { - "hashes": [ - "sha256:ef58335fe20e71384668159899985e6802bf668d1c82bb5ccd90592aba3b9ec2", - "sha256:adc95ec5eb0a04bc2aa8325e8a713b9aa82ccb30425f9efe0e4d7479e2fa3bcb" - ], - "version": "==0.4.2" - }, - "tornado": { - "hashes": [ - "sha256:62a5d4c66bf4e86d25a02e9de97293860b59e61f9c465e80336ba0fc308aacf6", - "sha256:f109c066411c44bcd3bc877267b45feb8e29092ede59dd0582739444c2344b00", - "sha256:e66f47db4753c6f6849af1f82f04bdc7d2c1f5d64b7cc11ddd17230295c8887f", - "sha256:2b40720a7b164848ca5c51fab0feac7e3717adcb87bec77f81f7809b72bf7f56", - "sha256:1fb8e494cd46c674d86fac5885a3ff87b0e283937a47d74eb3c02a48c9e89ad0" - ], - "version": "==4.5.2" } } } diff --git a/README.md b/README.md index 7981893..cdca809 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,23 @@ fedpkg and mbs-build). Currently, this consists of: -* `python -m fedmod rpm2module`: generates a draft modulemd file based on +* `fedmod rpm2module`: generates a draft modulemd file based on the given RPM name (multiple RPM names can be given, but the resulting draft module will lack any descriptive metadata in that case) -* `python -m fedmod fetch-metadata`: download the F27 Modular Server repository - metadata needed to generate draft module definitions +* `fedmod fetch-metadata`: download the F27 package and module metadata needed + to generate draft module definitions (the metadata sets to use are not yet + configurable) ## Project status `fedmod` is not yet released, but can already generate basic draft modules from a local development checkout: - $ pipenv run python -m fedmod rpm2module samba + $ pipenv run fedmod fetch-metadata + $ pipenv run fedmod rpm2module graphite-web -This will generate a draft `modulemd` file called `samba.yaml` in the current -directory. +This will generate a draft `modulemd` file called `graphite-web.yaml` in the +current directory. See the local development instructions below for info on setting up the execution environment with `pipenv`. @@ -50,20 +52,24 @@ whichever one you think looks better as your starting point. ## Modulemd creation -`python -m fedmod rpm2module [RPM NAMES]` creates a modulemd file from the +`fedmod rpm2module [RPM NAMES]` creates a modulemd file from the given package names. Output is written as `.yaml` when a single package name is given, and `modulemd-output.yaml` otherwise. The following metadata is currently used as input to the draft module generation process: -* Module definitions and package dependencies are pulled from the modular Fedora - Bikeshed repository, with the metadata being downloaded for local use via - the `python -m fedmod metadata` command +* Module definitions are pulled from the modular Fedora Bikeshed repository, + with the metadata being downloaded for local use via the + `fedmod fetch-metadata` command + +* Package dependency definitions are pulled from the regular Fedora 27 + release and updates repositories, with the metadata being downloaded for + local use via the `fedmod fetch-metadata` command * Descriptive metadata is taken from the system running `fedmod`. Due to this, `fedmod` currently only supports Fedora 26+. (This will be fixed to use - the same repository metadata as everything else) + the same repository metadata as is used for package dependency resolution) Module dependencies currently err on the side of making the generated modules smaller by permitting generated modules to depend on packages that aren't @@ -87,12 +93,15 @@ Other limitations in generated `modulemd` files: ### Runtime dependencies -The currently preferred means of dependency management is with `pipenv`: +The preferred dependency management tool for development is `pipenv`: $ pipenv --three --site-packages $ PIP_IGNORE_INSTALLED=1 pipenv install --dev -This will pull in the required Python level dependencies from PyPI. +The `PIP_IGNORE_INSTALLED=1` setting means that everything available to `pip` +will be installed into the virtual environment based on `Pipfile.lock`, and only +components that aren't installable with `pip` will be used from the system +Python installation. Some dependencies aren't currently available from PyPI, and hence need to be installed system-wide: @@ -100,31 +109,45 @@ installed system-wide: $ sudo dnf install python3-dnf python3-solv +### Additional development dependencies -### How to run tests +`pipenv` itself isn't packaged for Fedora yet, so the recommended bootstrapping +approach is to use the "pip script installer", `pipsi`: -To start a shell that's correctly configured to run the tests with the library -and all necessary dependencies installed: + $ sudo dnf install pipsi + $ pipsi install pew + $ pipsi install pipenv + +This will create a pair of isolated virtual environments in your home directory +specifically for `pipenv` and the tool it uses for virtual environment +management, `pew`. These can later be updated to newer versions using `pipsi`: + + $ pipsi upgrade pew + $ pipsi upgrade pipenv + +(Note: the `pipsi list` command will list all packages installed via `pipsi`, +and the commands they provide) + + +### Running the tests + +After going through the environment setup steps above, start a shell that's +correctly configured to run the tests with `fedmod` and all of its +dependencies available: - $ pipenv --three --site-packages - $ PIP_IGNORE_INSTALLED=1 pipenv install --dev $ pipenv shell -The `PIP_IGNORE_INSTALLED=1` setting means that everything available to `pip` -will be installed into the virtual environment based on `Pipfile.lock`, and only -components that aren't installable with `pip` will be used from the system -Python installation. +The metadata needed by the module generator tests can then be installed with +`fedmod` itself: -See the next section for additional preparation currently needed to run the -tests. + $ fedmod fetch-metadata The tests can then be run in the launched subshell with: $ pytest tests -Note: the tests will currently fail. Fixing that is a work in progress :) -### Reviewing dependencies +### Reviewing project dependencies To see the Python level dependencies graph: diff --git a/fedmod/__init__.py b/fedmod/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/fedmod/__init__.py +++ /dev/null diff --git a/fedmod/__main__.py b/fedmod/__main__.py deleted file mode 100644 index 9e88f92..0000000 --- a/fedmod/__main__.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from .cli import ModtoolsCLIHelper - - -if __name__ == "__main__": - cli = ModtoolsCLIHelper() - sys.exit(cli.run()) diff --git a/fedmod/_depchase.py b/fedmod/_depchase.py deleted file mode 100644 index 3323f95..0000000 --- a/fedmod/_depchase.py +++ /dev/null @@ -1,218 +0,0 @@ -import configparser -import itertools -import logging -import os -import sys -import tempfile -import smartcols -import solv - -from . import _repodata - -log = logging.getLogger(__name__) - -def get_module_for_rpm(rpm_name): - return _repodata._RPM_REVERSE_LOOKUP.get(rpm_name) - -def setup_pool(arch, repos=()): - pool = solv.Pool() - #pool.set_debuglevel(2) - pool.setarch(arch) - pool.set_loadcallback(_repodata.load_stub) - - for repo in repos: - repo.metadata_path = repo.metadata_path.format(arch=arch) - - for repo in repos: - assert repo.load(pool) - if "override" in repo.name: - repo.handle.priority = 99 - - addedprovides = pool.addfileprovides_queue() - if addedprovides: - for repo in repos: - repo.updateaddedprovides(addedprovides) - - pool.createwhatprovides() - - return pool - -def fix_deps(pool): - to_fix = ( - # Weak libcrypt-nss deps due to https://github.com/openSUSE/libsolv/issues/205 - ("glibc", solv.Selection.SELECTION_NAME, - solv.SOLVABLE_RECOMMENDS, lambda s: s.startswith("libcrypt-nss"), solv.SOLVABLE_SUGGESTS), - # Shim is not buildable - ("shim", solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_WITH_SOURCE, - solv.SOLVABLE_REQUIRES, lambda s: s in ("gnu-efi = 3.0w", "gnu-efi-devel = 3.0w"), None), - ) - for txt, flags, before, func, after in to_fix: - for s in pool.select(txt, flags).solvables(): - deps = s.lookup_deparray(before) - fixing = [dep for dep in deps if func(str(dep))] - for dep in fixing: - deps.remove(dep) - if after is not None: - s.add_deparray(after, dep) - # Use s.set_deparray() once will be available - s.unset(before) - for dep in deps: - s.add_deparray(before, dep) - -def get_sourcepkg(p, s=None, only_name=False): - if s is None: - s = p.lookup_sourcepkg()[:-4] - if only_name: - return s - # Let's try to find corresponding source - sel = p.pool.select(s, solv.Selection.SELECTION_CANON | solv.Selection.SELECTION_SOURCE_ONLY) - sel.filter(p.repo.appdata.srcrepo.handle.Selection()) - assert not sel.isempty(), "Could not find source package for {}".format(s) - solvables = sel.solvables() - assert len(solvables) == 1 - return solvables[0] - -def print_transaction(pool, transaction): - candq = transaction.newpackages() - if log.getEffectiveLevel() <= logging.INFO: - tb = smartcols.Table() - tb.title = "DEPENDENCY INFORMATION" - cl = tb.new_column("INFO") - cl.tree = True - cl_match = tb.new_column("MATCH") - for p in candq: - ln = tb.new_line() - ln[cl] = str(p) - for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES): - lns = tb.new_line(ln) - lns[cl] = str(dep) - matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep)) - if not matches and str(dep).startswith("/"): - # Append provides by files - # TODO: use Dataiterator for getting filelist - matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) - # It was possible to resolve set, so something is wrong here - assert matches - first = True - for m in matches: - if first: - lnc = lns - else: - lnss = tb.new_line(lns) - lnc = lnss - first = False - lnc[cl_match] = str(m) - log.info(tb) - -def _solve(solver, pkgnames): - """Given a set of package names, returns a list of solvables to install""" - pool = solver.pool - - # We have to =( - fix_deps(pool) - - jobs = [] - # Initial jobs, no conflicting packages - for n in pkgnames: - sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) - if sel.isempty(): - log.warn("Could not find package for {}".format(n)) - continue - jobs += sel.jobs(solv.Job.SOLVER_INSTALL) - problems = solver.solve(jobs) - if problems: - for problem in problems: - log.warn(problem) - - print_transaction(pool, solver.transaction()) - result = set() - for s in solver.transaction().newpackages(): - if s.name.startswith("fedora-release"): - # Relying on the F27 metadata injects irrelevant fedora-release deps - continue - if s.arch in ("src", "nosrc"): - continue - result.add(s) - return result - -def ensure_buildable(pkgset, pool=None): - """Given a set of solvables, returns a set of source packages & build deps""" - # The given package set may not be installable on its own - # That's OK, since other modules will provide those packages - # The goal of *this* method is to report the SRPMs that need to be - # built, and their build dependencies - sources = set(get_sourcepkg(s, only_name=True) for s in pkgset) - builddeps = ensure_installable(sources, pool=pool) - return sources, builddeps - -def make_pool(arch): - _repodata._populate_module_reverse_lookup() # TODO: Integrate this into the Pool abstraction - return setup_pool(arch, _repodata.setup_repos()) - -_DEFAULT_HINTS = ("glibc-minimal-langpack",) - -def ensure_installable(pkgnames, hints=_DEFAULT_HINTS, recommendations=False, pool=None): - """Iterate over the resolved dependency set for the given packages - - *hints*: Packages that have higher priority when more than one package - could satisfy a dependency. - *recommendations*: Whether or not to report recommended dependencies as well - as required dependencies (Default: required deps only) - """ - if pool is None: - pool = make_pool("x86_64") - # Set up initial hints - favorq = [] - for n in hints: - sel = pool.select(n, solv.Selection.SELECTION_NAME) - favorq += sel.jobs(solv.Job.SOLVER_FAVOR) - pool.setpooljobs(favorq) - - solver = pool.Solver() - if not recommendations: - # Ignore weak deps - solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1) - - return _solve(solver, pkgnames) - -def print_reldeps(pool, pkg): - sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) - assert not sel.isempty(), "Package can't be found" - found = sel.solvables() - assert len(found) == 1, "More matching solvables were found, {}".format(found) - s = found[0] - - reldep2str = {solv.SOLVABLE_REQUIRES: "requires", - solv.SOLVABLE_RECOMMENDS: "recommends", - solv.SOLVABLE_SUGGESTS: "suggests", - solv.SOLVABLE_SUPPLEMENTS: "supplements", - solv.SOLVABLE_ENHANCES: "enhances"} - for reltype, relstr in reldep2str.items(): - for dep in s.lookup_deparray(reltype): - print("{}: {}".format(relstr, dep)) - -def get_srpm_for_rpm(pool, pkg): - sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) - if sel.isempty(): - return f"unknown-component-{pkg}" - found = sel.solvables() - if len(found) > 1: - raise RuntimeError("More matching solvables were found, {}".format(found)) - s = found[0] - - return get_sourcepkg(s, only_name=True) - -def pkgs_by_sourcepkg(pool, sourcepkg): - - sel = pool.select(sourcepkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE) - if not sel.isempty(): - found = sel.solvables() - assert len(found) == 1, "More matching solvables were found, {}".format(found) - if found[0].arch in ("src", "nosrc"): - sourcepkg = str(found[0]) - else: - sourcepkg = get_sourcepkg(found[0], only_name=True) - - for p in (s for s in pool.solvables if s.arch not in ("src", "nosrc")): - if get_sourcepkg(p, only_name=True) == sourcepkg: - print(p) diff --git a/fedmod/_repodata.py b/fedmod/_repodata.py deleted file mode 100644 index 8eaa7ed..0000000 --- a/fedmod/_repodata.py +++ /dev/null @@ -1,393 +0,0 @@ -"""Helpers for metadata management""" -import sys -import tempfile -import os.path -import requests -import click -import gzip -import logging -import modulemd -import solv -from attr import attributes, attrib -from requests_toolbelt.downloadutils.tee import tee_to_file -from fnmatch import fnmatch -from urllib.parse import urljoin -from lxml import etree - -XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") -CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") - -log = logging.getLogger(__name__) - -FALLBACK_STREAM = 'master' -STREAM = 'f27' -ARCH = 'x86_64' -_F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" -_F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" -_F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" - -@attributes -class RepoPaths: - remote_repo_url = attrib(str) - remote_metadata_url = attrib(str) - local_cache_path = attrib(str) - local_metadata_path = attrib(str) - -def _define_repo(remote_prefix, local_cache_name, arch=None): - if arch is None: - local_arch_path = "source" - if "updates" in remote_prefix: - remote_arch_path = "SRPMS" - else: - remote_arch_path = "source/tree/" - else: - local_arch_path = arch - if "updates" in remote_prefix: - remote_arch_path = arch - else: - remote_arch_path = os.path.join(arch, "os/") - remote_repo_url = os.path.join(remote_prefix, remote_arch_path) - remote_metadata_url = os.path.join(remote_repo_url, "repodata/") - local_cache_path = os.path.join(CACHEDIR, "repos", local_cache_name, local_arch_path) - local_metadata_path = os.path.join(local_cache_path, "repodata/") - return RepoPaths(remote_repo_url, remote_metadata_url, - local_cache_path, local_metadata_path) - -_x86_64_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules", ARCH) -_SOURCE_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules") -_x86_64_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages", ARCH) -_SOURCE_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages") -_x86_64_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates", ARCH) -_SOURCE_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates") -_ALL_REPOS = ( - _x86_64_MODULE_INFO, - _SOURCE_MODULE_INFO, - _x86_64_PACKAGE_INFO, - _SOURCE_PACKAGE_INFO, - _x86_64_UPDATES_INFO, - _SOURCE_UPDATES_INFO, -) - -METADATA_SECTIONS = ("filelists", "primary", "modules") - -_REPOMD_XML_NAMESPACE = {"rpm": "http://linux.duke.edu/metadata/repo"} -def _read_repomd_location(repomd_xml, section): - location = repomd_xml.find(f"rpm:data[@type='{section}']/rpm:location", _REPOMD_XML_NAMESPACE) - if location is not None: - return location.attrib["href"] - return None - -def _download_one_file(remote_url, filename): - if os.path.exists(filename) and not filename.endswith("repomd.xml"): - print(f" Skipping download; {filename} already exists") - return - with requests.get(remote_url, stream=True) as response: - print(f" Downloading {remote_url}") - chunksize = 65536 - expected_chunks = int(response.headers["content-length"]) / chunksize - downloader = tee_to_file(response, filename=filename, chunksize=chunksize) - show_progress = click.progressbar(downloader, length=expected_chunks) - with show_progress: - for chunk in show_progress: - pass - print(f" Added {filename} to cache") - -def _download_metadata_files(repo_paths): - local_path = repo_paths.local_cache_path - local_metadata_path = repo_paths.local_metadata_path - os.makedirs(local_metadata_path, exist_ok=True) - repomd_url = urljoin(repo_paths.remote_metadata_url, "repomd.xml") - print(f"Remote metadata: {repomd_url}") - response = requests.get(repomd_url) - response.raise_for_status() - repomd_filename = os.path.join(local_metadata_path, "repomd.xml") - with open(repomd_filename, "wb") as f: - f.write(response.content) - print(f" Cached metadata in {repomd_filename}") - repomd_xml = etree.parse(repomd_filename) - files_to_fetch = set() - for section in METADATA_SECTIONS: - relative_href = _read_repomd_location(repomd_xml, section) - if relative_href is not None: - files_to_fetch.add(relative_href) - predownload = set(os.listdir(local_path)) - for relative_href in files_to_fetch: - absolute_href = urljoin(repo_paths.remote_repo_url, relative_href) - filename = os.path.join(local_path, relative_href) - # This could be parallelised with concurrent.futures, but - # probably not worth it (it makes the progress bars trickier) - _download_one_file(absolute_href, filename) - postdownload = set(os.listdir(local_path)) - # Prune any old metadata files automatically - if len(postdownload) >= (len(predownload) + len(METADATA_SECTIONS)): - # TODO: Actually prune old metadata files - pass - - -def download_repo_metadata(): - """Downloads the latest repo metadata""" - for repo_definition in _ALL_REPOS: - _download_metadata_files(repo_definition) - -_SRPM_REVERSE_LOOKUP = {} # SRPM name : module name -_RPM_REVERSE_LOOKUP = {} # RPM name : module name -def _populate_module_reverse_lookup(): - # TODO: Cache the reverse mapping in _depchase.Repo instances, as with the solver data - metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) - repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") - repomd_xml = etree.parse(repomd_fname) - repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") - if repo_relative_modulemd is None: - raise RuntimeError(f"No 'modules' entry found in {repomd_fname}. Is the metadata for a non-modular repo?") - repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) - with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: - modules_yaml = modules_yaml_gz.read() - modules = modulemd.loads_all(modules_yaml) - for module in modules: - for srpmname in module.components.rpms: - # This isn't entirely valid, as it doesn't account for multiple - # modules that include the same source RPM with different output - # filters (e.g. python3-ecosystem vs python2-ecosystem) - _SRPM_REVERSE_LOOKUP[srpmname] = module.name - for rpmname in module.artifacts.rpms: - # This is only valid for module sets that are guaranteed to be - # fully coinstallable, and hence only allow any given RPM to be - # published by at most one module - rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] - _RPM_REVERSE_LOOKUP[rpmprefix] = module.name - -class Repo(object): - def __init__(self, name, metadata_path): - self.name = name - self.metadata_path = metadata_path - self.handle = None - self.cookie = None - self.extcookie = None - self.srcrepo = None - - @staticmethod - def calc_cookie_fp(fp): - chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) - chksum.add("1.1") - chksum.add_fp(fp) - return chksum.raw() - - @staticmethod - def calc_cookie_ext(f, cookie): - chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) - chksum.add("1.1") - chksum.add(cookie) - chksum.add_fstat(f.fileno()) - return chksum.raw() - - def cachepath(self, ext=None): - path = "{}-{}".format(self.name.replace(".", "_"), self.metadata_path) - if ext: - path = "{}-{}.solvx".format(path, ext) - else: - path = "{}.solv".format(path) - return os.path.join(CACHEDIR, path.replace("/", "_")) - - def usecachedrepo(self, ext, mark=False): - try: - repopath = self.cachepath(ext) - f = open(repopath, "rb") - f.seek(-32, os.SEEK_END) - fcookie = f.read(32) - if len(fcookie) != 32: - return False - cookie = self.extcookie if ext else self.cookie - if cookie and fcookie != cookie: - return False - if not ext: - f.seek(-32 * 2, os.SEEK_END) - fextcookie = f.read(32) - if len(fextcookie) != 32: - return False - f.seek(0) - f = solv.xfopen_fd(None, f.fileno()) - flags = 0 - if ext: - flags = solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES - if ext != "DL": - flags |= solv.Repo.REPO_LOCALPOOL - if not self.handle.add_solv(f, flags): - return False - if not ext: - self.cookie = fcookie - self.extcookie = fextcookie - if mark: - # no futimes in python? - try: - os.utime(repopath, None) - except Exception: - pass - except IOError: - return False - return True - - def writecachedrepo(self, ext, repodata=None): - tmpname = None - try: - if not os.path.isdir(CACHEDIR): - os.mkdir(CACHEDIR, 0o755) - fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", dir=CACHEDIR) - os.fchmod(fd, 0o444) - f = os.fdopen(fd, "wb+") - f = solv.xfopen_fd(None, f.fileno()) - if not repodata: - self.handle.write(f) - elif ext: - repodata.write(f) - else: - # rewrite_repos case, do not write stubs - self.handle.write_first_repodata(f) - f.flush() - if not ext: - if not self.extcookie: - self.extcookie = self.calc_cookie_ext(f, self.cookie) - f.write(self.extcookie) - if not ext: - f.write(self.cookie) - else: - f.write(self.extcookie) - f.close - if self.handle.iscontiguous(): - # switch to saved repo to activate paging and save memory - nf = solv.xfopen(tmpname) - if not ext: - # main repo - self.handle.empty() - flags = solv.Repo.SOLV_ADD_NO_STUBS - if repodata: - # rewrite repos case, recreate stubs - flags = 0 - if not self.handle.add_solv(nf, flags): - sys.exit("internal error, cannot reload solv file") - else: - # extension repodata - # need to extend to repo boundaries, as this is how - # repodata.write() has written the data - repodata.extend_to_repo() - flags = solv.Repo.REPO_EXTEND_SOLVABLES - if ext != "DL": - flags |= solv.Repo.REPO_LOCALPOOL - repodata.add_solv(nf, flags) - os.rename(tmpname, self.cachepath(ext)) - except (OSError, IOError): - if tmpname: - os.unlink(tmpname) - - def load(self, pool): - assert not self.handle - self.handle = pool.add_repo(self.name) - self.handle.appdata = self - f = self.read_repo_metadata("repodata/repomd.xml", False, None) - if not f: - self.handle.free(True) - self.handle = None - return False - self.cookie = self.calc_cookie_fp(f) - if self.usecachedrepo(None, True): - return True - self.handle.add_repomdxml(f) - fname, fchksum = self.find("primary") - if not fname: - return False - f = self.read_repo_metadata(fname, True, fchksum) - if not f: - return False - self.handle.add_rpmmd(f, None) - self.add_exts() - self.writecachedrepo(None) - # Must be called after writing the repo - self.handle.create_stubs() - return True - - def read_repo_metadata(self, fname, uncompress, chksum): - f = open("{}/{}".format(self.metadata_path, fname)) - return solv.xfopen_fd(fname if uncompress else None, f.fileno()) - - def find(self, what): - di = self.handle.Dataiterator_meta(solv.REPOSITORY_REPOMD_TYPE, what, solv.Dataiterator.SEARCH_STRING) - di.prepend_keyname(solv.REPOSITORY_REPOMD) - for d in di: - dp = d.parentpos() - filename = dp.lookup_str(solv.REPOSITORY_REPOMD_LOCATION) - chksum = dp.lookup_checksum(solv.REPOSITORY_REPOMD_CHECKSUM) - if filename: - if not chksum: - print("No {} file checksum!".format(filename)) - return filename, chksum - return None, None - - def add_ext_keys(self, ext, repodata, handle): - if ext == "FL": - repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.SOLVABLE_FILELIST) - repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.REPOKEY_TYPE_DIRSTRARRAY) - else: - raise NotImplementedError - - def add_ext(self, repodata, what, ext): - filename, chksum = self.find(what) - if not filename: - return - handle = repodata.new_handle() - repodata.set_poolstr(handle, solv.REPOSITORY_REPOMD_TYPE, what) - repodata.set_str(handle, solv.REPOSITORY_REPOMD_LOCATION, filename) - repodata.set_checksum(handle, solv.REPOSITORY_REPOMD_CHECKSUM, chksum) - self.add_ext_keys(ext, repodata, handle) - repodata.add_flexarray(solv.SOLVID_META, solv.REPOSITORY_EXTERNAL, handle) - - def add_exts(self): - repodata = self.handle.add_repodata() - self.add_ext(repodata, "filelists", "FL") - repodata.internalize() - - def load_ext(self, repodata): - repomdtype = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_TYPE) - if repomdtype == "filelists": - ext = "FL" - else: - assert False - if self.usecachedrepo(ext): - return True - filename = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_LOCATION) - filechksum = repodata.lookup_checksum(solv.SOLVID_META, solv.REPOSITORY_REPOMD_CHECKSUM) - f = self.read_repo_metadata(filename, True, filechksum) - if not f: - return False - if ext == "FL": - self.handle.add_rpmmd(f, "FL", solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES | solv.Repo.REPO_LOCALPOOL) - self.writecachedrepo(ext, repodata) - return True - - def updateaddedprovides(self, addedprovides): - if self.handle.isempty(): - return - # make sure there's just one real repodata with extensions - repodata = self.handle.first_repodata() - if not repodata: - return - oldaddedprovides = repodata.lookup_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES) - if not set(addedprovides) <= set(oldaddedprovides): - for id in addedprovides: - repodata.add_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES, id) - repodata.internalize() - self.writecachedrepo(None, repodata) - -def load_stub(repodata): - repo = repodata.repo.appdata - if repo: - return repo.load_ext(repodata) - return False - -def setup_repos(): - - srcrepo = Repo("f27-source", _SOURCE_PACKAGE_INFO.local_cache_path) - repo = Repo("f27", _x86_64_PACKAGE_INFO.local_cache_path) - repo.srcrepo = srcrepo - updates_srcrepo = Repo("f27-updates-source", _SOURCE_UPDATES_INFO.local_cache_path) - updates_repo = Repo("f27-updates", _x86_64_UPDATES_INFO.local_cache_path) - updates_repo.srcrepo = updates_srcrepo - return [repo, srcrepo, updates_repo, updates_srcrepo] diff --git a/fedmod/cli.py b/fedmod/cli.py deleted file mode 100644 index 49b3206..0000000 --- a/fedmod/cli.py +++ /dev/null @@ -1,75 +0,0 @@ -import sys -import argparse -import logging - -from .module_generator import ModuleGenerator -from . import _depchase, _repodata - -# TODO: Switch this over to click (already a dependency for progress bars) - -class ModtoolsCLI(object): - """ Class for processing data from commandline """ - - @staticmethod - def build_parser(): - parser = argparse.ArgumentParser(description="Generates module related files") - base_parser = argparse.ArgumentParser(add_help=False) - base_parser.add_argument( - "--verbose", - "-v", - action="store_true", - help="verbose operation" - ) - - subparsers = parser.add_subparsers(dest="cmd_name") - - parser_rpm2module = subparsers.add_parser( - 'rpm2module', parents=[base_parser], - help="Generates modulemd file", - description="Gets package info and dependencies and creates modulemd file." - ) - parser_rpm2module.add_argument( - "pkgs", - metavar='PKGS', - nargs='+', - help="Specify list of packages for module.", - ) - - parser_metadata = subparsers.add_parser( - 'fetch-metadata', parents=[base_parser], - help="Fetches repository metadata", - description="Caches needed repository metadata locally" - ) - - return parser - - def __init__(self, args=None): - self.parser = ModtoolsCLI.build_parser() - self.args = self.parser.parse_args(args) - - def __getattr__(self, name): - try: - return getattr(self.args, name) - except AttributeError: - return object.__getattribute__(self, name) - - -class ModtoolsCLIHelper(object): - - @staticmethod - def run(): - try: - cli = ModtoolsCLI(sys.argv[1:]) - if cli.args.verbose: - logging.basicConfig(level=logging.INFO) - if cli.args.cmd_name == 'rpm2module': - mg = ModuleGenerator(cli.args.pkgs) - mg.run() - elif cli.args.cmd_name == 'fetch-metadata': - _repodata.download_repo_metadata() - - except KeyboardInterrupt: - print('\nInterrupted by user') - except Exception as e: - logging.exception("Unexpected exception") - sys.exit(1) diff --git a/fedmod/module_generator.py b/fedmod/module_generator.py deleted file mode 100644 index 33836db..0000000 --- a/fedmod/module_generator.py +++ /dev/null @@ -1,151 +0,0 @@ -from __future__ import absolute_import - -import modulemd -import logging -import dnf -from . import _depchase - -def _name_only(rpm_name): - name, version, release = rpm_name.rsplit("-", 2) - return name - -def _categorise_deps(pool, all_rpm_deps): - module_deps = set() - remaining_rpm_deps = set() - for pkg in all_rpm_deps: - modname = _depchase.get_module_for_rpm(pkg.name) - if modname is not None: - module_deps.add(modname) - else: - remaining_rpm_deps.add(pkg) - return module_deps, remaining_rpm_deps - -class ModuleGenerator(object): - - def __init__(self, pkgs): - self.pkgs = pkgs - self.pkg = None - self.mmd = modulemd.ModuleMetadata() - self._calculate_dependencies() - - def _calculate_dependencies(self): - pkgs = self.pkgs - pool = _depchase.make_pool("x86_64") - self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} - run_deps = _depchase.ensure_installable(pkgs, pool=pool) - module_run_deps, rpm_run_deps = _categorise_deps(pool, run_deps) - self.module_run_deps = module_run_deps - run_srpms, build_deps = _depchase.ensure_buildable(rpm_run_deps) - module_build_deps = set() - resolved_build_deps = set() - build_srpms = set() - for i in range(10): - # Arbitrary bound of 10 levels of SRPM bootstrapping - new_module_build_deps, remaining_build_deps = _categorise_deps(pool, build_deps) - module_build_deps |= new_module_build_deps - resolved_build_deps |= (build_deps - remaining_build_deps) - build_deps -= resolved_build_deps - if build_deps: - new_build_srpms, remaining_build_deps = _depchase.ensure_buildable(build_deps) - build_srpms |= new_build_srpms - resolved_build_deps |= (build_deps - remaining_build_deps) - build_deps -= resolved_build_deps - if not build_deps: - break - else: - logging.warn("Failed to close out build dependencies after 10 iteration") - self.module_run_deps = module_run_deps - self.module_build_deps = module_build_deps - run_srpm_names = {_name_only(n) for n in run_srpms} - build_srpm_names = {_name_only(n) for n in build_srpms} - self.run_srpms = run_srpm_names - build_srpm_names - self.build_srpms = build_srpm_names - run_srpm_names - self.build_and_run_srpms = run_srpm_names & build_srpm_names - self.unresolved_build_rpms = {_name_only(s.name) for s in build_deps} - - - def _get_pkg_info(self): - """Function loads package from dnf""" - # TODO: Get this from the _depchase metadata, not the system metadata - logging.info("Getting package info from DNF") - b = dnf.Base() - b.read_all_repos() - b.fill_sack() - q = b.sack.query().filter(name=self.pkgs, reponame='fedora', latest=True) - if len(q) > 1: - raise ValueError('Name of package is not unique') - if len(q) == 0: - raise ValueError('No package found in repo') - self.pkg = q[0] - - def _save_module_md(self): - """ - Function saves modulemd file to the current directory - based on argument name - :return: - """ - - if len(self.pkgs) == 1: - file_name = self.pkgs[0] + '.yaml' - else: - file_name = "modulemd-output.yaml" - self.mmd.dump(file_name) - print('Modulemd file is generated here ./%s' % file_name) - return True - - def _update_module_md(self): - """ - Function updates modulemd file with dependencies - are information taken from SPEC file. - :return: - """ - self.mmd.add_module_license("MIT") - - if len(self.pkgs) == 1: - self.mmd.summary = str(self.pkg.summary) - self.mmd.description = str(self.pkg.description) - - # Default license for the module metadata, same as default Fedora - # content license. - - self.mmd.add_content_license(str(self.pkg.license)) - - # Declare the public API - for pkg in self.api_srpms: - self.mmd.api.add_rpm(pkg) - self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg)) - - # Declare module level dependencies - for modname in self.module_build_deps: - self.mmd.buildrequires[modname] = "f27" - for modname in self.module_run_deps: - self.mmd.requires[modname] = "f27" - - # Add any other RPMs not available from existing modules as components - for pkg in self.build_and_run_srpms: - self.mmd.components.add_rpm(pkg, "Build and runtime dependency.", buildorder=self._get_build_order(pkg)) - - for pkg in self.run_srpms: - self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg)) - - for pkg in self.build_srpms: - self.mmd.components.add_rpm(pkg, "Build dependency.", buildorder=self._get_build_order(pkg)) - # Filter out any build-only packages - # TODO: This won't filter out all the RPMs, only the one matching the SRPM name - # See https://pagure.io/modulemd/issue/54 for discussion - self.mmd.filter.add_rpm(pkg) - - # TODO: Emit something for non-empty self.unresolved_build_rpms - - - def _get_build_order(self, pkg): - if pkg in self.mmd.api.rpms: - return 10 - else: - return 0 - - def run(self): - if len(self.pkgs) == 1: - self._get_pkg_info() - self._update_module_md() - self._save_module_md() diff --git a/setup.py b/setup.py deleted file mode 100644 index 5757383..0000000 --- a/setup.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -from setuptools import setup, find_packages - -setup( - name='fedmod', - version='0.0.1', - author='Dominika Hodovska', - author_email='dhodovsk@redhat.com', - maintainer='Nick Coghlan', - maintainer_email='ncoghlan@redhat.com', - description='Utilities for generating & maintaining modulemd files', - long_description=( - "fedmod provides tools for converting existing RPMs (most notably " - "metapackages) into module definitions in Fedora's modulemd format." - ), - license='MIT', - keywords='modularization modularity module modulemd fedora', - url='https://pagure.io/modularity/fedmod', - entry_point={ - 'console_scripts': [ - 'fedmod=modularity.cli.ModtoolsCliHelper.run' - ] - }, - install_requires=[ - 'modulemd', - 'click', - 'requests', - 'requests-toolbelt', - 'lxml', - ], - packages=find_packages(), -) diff --git a/src/MANIFEST.in b/src/MANIFEST.in new file mode 100644 index 0000000..b52089f --- /dev/null +++ b/src/MANIFEST.in @@ -0,0 +1,5 @@ +include MANIFEST.in +include README.md +include LICENSE +include modtools +recursive-include modularity * diff --git a/src/_fedmod/__init__.py b/src/_fedmod/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/_fedmod/__init__.py diff --git a/src/_fedmod/__main__.py b/src/_fedmod/__main__.py new file mode 100644 index 0000000..9e88f92 --- /dev/null +++ b/src/_fedmod/__main__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +import sys +from .cli import ModtoolsCLIHelper + + +if __name__ == "__main__": + cli = ModtoolsCLIHelper() + sys.exit(cli.run()) diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py new file mode 100644 index 0000000..3323f95 --- /dev/null +++ b/src/_fedmod/_depchase.py @@ -0,0 +1,218 @@ +import configparser +import itertools +import logging +import os +import sys +import tempfile +import smartcols +import solv + +from . import _repodata + +log = logging.getLogger(__name__) + +def get_module_for_rpm(rpm_name): + return _repodata._RPM_REVERSE_LOOKUP.get(rpm_name) + +def setup_pool(arch, repos=()): + pool = solv.Pool() + #pool.set_debuglevel(2) + pool.setarch(arch) + pool.set_loadcallback(_repodata.load_stub) + + for repo in repos: + repo.metadata_path = repo.metadata_path.format(arch=arch) + + for repo in repos: + assert repo.load(pool) + if "override" in repo.name: + repo.handle.priority = 99 + + addedprovides = pool.addfileprovides_queue() + if addedprovides: + for repo in repos: + repo.updateaddedprovides(addedprovides) + + pool.createwhatprovides() + + return pool + +def fix_deps(pool): + to_fix = ( + # Weak libcrypt-nss deps due to https://github.com/openSUSE/libsolv/issues/205 + ("glibc", solv.Selection.SELECTION_NAME, + solv.SOLVABLE_RECOMMENDS, lambda s: s.startswith("libcrypt-nss"), solv.SOLVABLE_SUGGESTS), + # Shim is not buildable + ("shim", solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_WITH_SOURCE, + solv.SOLVABLE_REQUIRES, lambda s: s in ("gnu-efi = 3.0w", "gnu-efi-devel = 3.0w"), None), + ) + for txt, flags, before, func, after in to_fix: + for s in pool.select(txt, flags).solvables(): + deps = s.lookup_deparray(before) + fixing = [dep for dep in deps if func(str(dep))] + for dep in fixing: + deps.remove(dep) + if after is not None: + s.add_deparray(after, dep) + # Use s.set_deparray() once will be available + s.unset(before) + for dep in deps: + s.add_deparray(before, dep) + +def get_sourcepkg(p, s=None, only_name=False): + if s is None: + s = p.lookup_sourcepkg()[:-4] + if only_name: + return s + # Let's try to find corresponding source + sel = p.pool.select(s, solv.Selection.SELECTION_CANON | solv.Selection.SELECTION_SOURCE_ONLY) + sel.filter(p.repo.appdata.srcrepo.handle.Selection()) + assert not sel.isempty(), "Could not find source package for {}".format(s) + solvables = sel.solvables() + assert len(solvables) == 1 + return solvables[0] + +def print_transaction(pool, transaction): + candq = transaction.newpackages() + if log.getEffectiveLevel() <= logging.INFO: + tb = smartcols.Table() + tb.title = "DEPENDENCY INFORMATION" + cl = tb.new_column("INFO") + cl.tree = True + cl_match = tb.new_column("MATCH") + for p in candq: + ln = tb.new_line() + ln[cl] = str(p) + for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES): + lns = tb.new_line(ln) + lns[cl] = str(dep) + matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep)) + if not matches and str(dep).startswith("/"): + # Append provides by files + # TODO: use Dataiterator for getting filelist + matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) + # It was possible to resolve set, so something is wrong here + assert matches + first = True + for m in matches: + if first: + lnc = lns + else: + lnss = tb.new_line(lns) + lnc = lnss + first = False + lnc[cl_match] = str(m) + log.info(tb) + +def _solve(solver, pkgnames): + """Given a set of package names, returns a list of solvables to install""" + pool = solver.pool + + # We have to =( + fix_deps(pool) + + jobs = [] + # Initial jobs, no conflicting packages + for n in pkgnames: + sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) + if sel.isempty(): + log.warn("Could not find package for {}".format(n)) + continue + jobs += sel.jobs(solv.Job.SOLVER_INSTALL) + problems = solver.solve(jobs) + if problems: + for problem in problems: + log.warn(problem) + + print_transaction(pool, solver.transaction()) + result = set() + for s in solver.transaction().newpackages(): + if s.name.startswith("fedora-release"): + # Relying on the F27 metadata injects irrelevant fedora-release deps + continue + if s.arch in ("src", "nosrc"): + continue + result.add(s) + return result + +def ensure_buildable(pkgset, pool=None): + """Given a set of solvables, returns a set of source packages & build deps""" + # The given package set may not be installable on its own + # That's OK, since other modules will provide those packages + # The goal of *this* method is to report the SRPMs that need to be + # built, and their build dependencies + sources = set(get_sourcepkg(s, only_name=True) for s in pkgset) + builddeps = ensure_installable(sources, pool=pool) + return sources, builddeps + +def make_pool(arch): + _repodata._populate_module_reverse_lookup() # TODO: Integrate this into the Pool abstraction + return setup_pool(arch, _repodata.setup_repos()) + +_DEFAULT_HINTS = ("glibc-minimal-langpack",) + +def ensure_installable(pkgnames, hints=_DEFAULT_HINTS, recommendations=False, pool=None): + """Iterate over the resolved dependency set for the given packages + + *hints*: Packages that have higher priority when more than one package + could satisfy a dependency. + *recommendations*: Whether or not to report recommended dependencies as well + as required dependencies (Default: required deps only) + """ + if pool is None: + pool = make_pool("x86_64") + # Set up initial hints + favorq = [] + for n in hints: + sel = pool.select(n, solv.Selection.SELECTION_NAME) + favorq += sel.jobs(solv.Job.SOLVER_FAVOR) + pool.setpooljobs(favorq) + + solver = pool.Solver() + if not recommendations: + # Ignore weak deps + solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1) + + return _solve(solver, pkgnames) + +def print_reldeps(pool, pkg): + sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) + assert not sel.isempty(), "Package can't be found" + found = sel.solvables() + assert len(found) == 1, "More matching solvables were found, {}".format(found) + s = found[0] + + reldep2str = {solv.SOLVABLE_REQUIRES: "requires", + solv.SOLVABLE_RECOMMENDS: "recommends", + solv.SOLVABLE_SUGGESTS: "suggests", + solv.SOLVABLE_SUPPLEMENTS: "supplements", + solv.SOLVABLE_ENHANCES: "enhances"} + for reltype, relstr in reldep2str.items(): + for dep in s.lookup_deparray(reltype): + print("{}: {}".format(relstr, dep)) + +def get_srpm_for_rpm(pool, pkg): + sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) + if sel.isempty(): + return f"unknown-component-{pkg}" + found = sel.solvables() + if len(found) > 1: + raise RuntimeError("More matching solvables were found, {}".format(found)) + s = found[0] + + return get_sourcepkg(s, only_name=True) + +def pkgs_by_sourcepkg(pool, sourcepkg): + + sel = pool.select(sourcepkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE) + if not sel.isempty(): + found = sel.solvables() + assert len(found) == 1, "More matching solvables were found, {}".format(found) + if found[0].arch in ("src", "nosrc"): + sourcepkg = str(found[0]) + else: + sourcepkg = get_sourcepkg(found[0], only_name=True) + + for p in (s for s in pool.solvables if s.arch not in ("src", "nosrc")): + if get_sourcepkg(p, only_name=True) == sourcepkg: + print(p) diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py new file mode 100644 index 0000000..8eaa7ed --- /dev/null +++ b/src/_fedmod/_repodata.py @@ -0,0 +1,393 @@ +"""Helpers for metadata management""" +import sys +import tempfile +import os.path +import requests +import click +import gzip +import logging +import modulemd +import solv +from attr import attributes, attrib +from requests_toolbelt.downloadutils.tee import tee_to_file +from fnmatch import fnmatch +from urllib.parse import urljoin +from lxml import etree + +XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") +CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") + +log = logging.getLogger(__name__) + +FALLBACK_STREAM = 'master' +STREAM = 'f27' +ARCH = 'x86_64' +_F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" +_F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" +_F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" + +@attributes +class RepoPaths: + remote_repo_url = attrib(str) + remote_metadata_url = attrib(str) + local_cache_path = attrib(str) + local_metadata_path = attrib(str) + +def _define_repo(remote_prefix, local_cache_name, arch=None): + if arch is None: + local_arch_path = "source" + if "updates" in remote_prefix: + remote_arch_path = "SRPMS" + else: + remote_arch_path = "source/tree/" + else: + local_arch_path = arch + if "updates" in remote_prefix: + remote_arch_path = arch + else: + remote_arch_path = os.path.join(arch, "os/") + remote_repo_url = os.path.join(remote_prefix, remote_arch_path) + remote_metadata_url = os.path.join(remote_repo_url, "repodata/") + local_cache_path = os.path.join(CACHEDIR, "repos", local_cache_name, local_arch_path) + local_metadata_path = os.path.join(local_cache_path, "repodata/") + return RepoPaths(remote_repo_url, remote_metadata_url, + local_cache_path, local_metadata_path) + +_x86_64_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules", ARCH) +_SOURCE_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules") +_x86_64_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages", ARCH) +_SOURCE_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages") +_x86_64_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates", ARCH) +_SOURCE_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates") +_ALL_REPOS = ( + _x86_64_MODULE_INFO, + _SOURCE_MODULE_INFO, + _x86_64_PACKAGE_INFO, + _SOURCE_PACKAGE_INFO, + _x86_64_UPDATES_INFO, + _SOURCE_UPDATES_INFO, +) + +METADATA_SECTIONS = ("filelists", "primary", "modules") + +_REPOMD_XML_NAMESPACE = {"rpm": "http://linux.duke.edu/metadata/repo"} +def _read_repomd_location(repomd_xml, section): + location = repomd_xml.find(f"rpm:data[@type='{section}']/rpm:location", _REPOMD_XML_NAMESPACE) + if location is not None: + return location.attrib["href"] + return None + +def _download_one_file(remote_url, filename): + if os.path.exists(filename) and not filename.endswith("repomd.xml"): + print(f" Skipping download; {filename} already exists") + return + with requests.get(remote_url, stream=True) as response: + print(f" Downloading {remote_url}") + chunksize = 65536 + expected_chunks = int(response.headers["content-length"]) / chunksize + downloader = tee_to_file(response, filename=filename, chunksize=chunksize) + show_progress = click.progressbar(downloader, length=expected_chunks) + with show_progress: + for chunk in show_progress: + pass + print(f" Added {filename} to cache") + +def _download_metadata_files(repo_paths): + local_path = repo_paths.local_cache_path + local_metadata_path = repo_paths.local_metadata_path + os.makedirs(local_metadata_path, exist_ok=True) + repomd_url = urljoin(repo_paths.remote_metadata_url, "repomd.xml") + print(f"Remote metadata: {repomd_url}") + response = requests.get(repomd_url) + response.raise_for_status() + repomd_filename = os.path.join(local_metadata_path, "repomd.xml") + with open(repomd_filename, "wb") as f: + f.write(response.content) + print(f" Cached metadata in {repomd_filename}") + repomd_xml = etree.parse(repomd_filename) + files_to_fetch = set() + for section in METADATA_SECTIONS: + relative_href = _read_repomd_location(repomd_xml, section) + if relative_href is not None: + files_to_fetch.add(relative_href) + predownload = set(os.listdir(local_path)) + for relative_href in files_to_fetch: + absolute_href = urljoin(repo_paths.remote_repo_url, relative_href) + filename = os.path.join(local_path, relative_href) + # This could be parallelised with concurrent.futures, but + # probably not worth it (it makes the progress bars trickier) + _download_one_file(absolute_href, filename) + postdownload = set(os.listdir(local_path)) + # Prune any old metadata files automatically + if len(postdownload) >= (len(predownload) + len(METADATA_SECTIONS)): + # TODO: Actually prune old metadata files + pass + + +def download_repo_metadata(): + """Downloads the latest repo metadata""" + for repo_definition in _ALL_REPOS: + _download_metadata_files(repo_definition) + +_SRPM_REVERSE_LOOKUP = {} # SRPM name : module name +_RPM_REVERSE_LOOKUP = {} # RPM name : module name +def _populate_module_reverse_lookup(): + # TODO: Cache the reverse mapping in _depchase.Repo instances, as with the solver data + metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) + repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") + repomd_xml = etree.parse(repomd_fname) + repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") + if repo_relative_modulemd is None: + raise RuntimeError(f"No 'modules' entry found in {repomd_fname}. Is the metadata for a non-modular repo?") + repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) + with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: + modules_yaml = modules_yaml_gz.read() + modules = modulemd.loads_all(modules_yaml) + for module in modules: + for srpmname in module.components.rpms: + # This isn't entirely valid, as it doesn't account for multiple + # modules that include the same source RPM with different output + # filters (e.g. python3-ecosystem vs python2-ecosystem) + _SRPM_REVERSE_LOOKUP[srpmname] = module.name + for rpmname in module.artifacts.rpms: + # This is only valid for module sets that are guaranteed to be + # fully coinstallable, and hence only allow any given RPM to be + # published by at most one module + rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] + _RPM_REVERSE_LOOKUP[rpmprefix] = module.name + +class Repo(object): + def __init__(self, name, metadata_path): + self.name = name + self.metadata_path = metadata_path + self.handle = None + self.cookie = None + self.extcookie = None + self.srcrepo = None + + @staticmethod + def calc_cookie_fp(fp): + chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) + chksum.add("1.1") + chksum.add_fp(fp) + return chksum.raw() + + @staticmethod + def calc_cookie_ext(f, cookie): + chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) + chksum.add("1.1") + chksum.add(cookie) + chksum.add_fstat(f.fileno()) + return chksum.raw() + + def cachepath(self, ext=None): + path = "{}-{}".format(self.name.replace(".", "_"), self.metadata_path) + if ext: + path = "{}-{}.solvx".format(path, ext) + else: + path = "{}.solv".format(path) + return os.path.join(CACHEDIR, path.replace("/", "_")) + + def usecachedrepo(self, ext, mark=False): + try: + repopath = self.cachepath(ext) + f = open(repopath, "rb") + f.seek(-32, os.SEEK_END) + fcookie = f.read(32) + if len(fcookie) != 32: + return False + cookie = self.extcookie if ext else self.cookie + if cookie and fcookie != cookie: + return False + if not ext: + f.seek(-32 * 2, os.SEEK_END) + fextcookie = f.read(32) + if len(fextcookie) != 32: + return False + f.seek(0) + f = solv.xfopen_fd(None, f.fileno()) + flags = 0 + if ext: + flags = solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES + if ext != "DL": + flags |= solv.Repo.REPO_LOCALPOOL + if not self.handle.add_solv(f, flags): + return False + if not ext: + self.cookie = fcookie + self.extcookie = fextcookie + if mark: + # no futimes in python? + try: + os.utime(repopath, None) + except Exception: + pass + except IOError: + return False + return True + + def writecachedrepo(self, ext, repodata=None): + tmpname = None + try: + if not os.path.isdir(CACHEDIR): + os.mkdir(CACHEDIR, 0o755) + fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", dir=CACHEDIR) + os.fchmod(fd, 0o444) + f = os.fdopen(fd, "wb+") + f = solv.xfopen_fd(None, f.fileno()) + if not repodata: + self.handle.write(f) + elif ext: + repodata.write(f) + else: + # rewrite_repos case, do not write stubs + self.handle.write_first_repodata(f) + f.flush() + if not ext: + if not self.extcookie: + self.extcookie = self.calc_cookie_ext(f, self.cookie) + f.write(self.extcookie) + if not ext: + f.write(self.cookie) + else: + f.write(self.extcookie) + f.close + if self.handle.iscontiguous(): + # switch to saved repo to activate paging and save memory + nf = solv.xfopen(tmpname) + if not ext: + # main repo + self.handle.empty() + flags = solv.Repo.SOLV_ADD_NO_STUBS + if repodata: + # rewrite repos case, recreate stubs + flags = 0 + if not self.handle.add_solv(nf, flags): + sys.exit("internal error, cannot reload solv file") + else: + # extension repodata + # need to extend to repo boundaries, as this is how + # repodata.write() has written the data + repodata.extend_to_repo() + flags = solv.Repo.REPO_EXTEND_SOLVABLES + if ext != "DL": + flags |= solv.Repo.REPO_LOCALPOOL + repodata.add_solv(nf, flags) + os.rename(tmpname, self.cachepath(ext)) + except (OSError, IOError): + if tmpname: + os.unlink(tmpname) + + def load(self, pool): + assert not self.handle + self.handle = pool.add_repo(self.name) + self.handle.appdata = self + f = self.read_repo_metadata("repodata/repomd.xml", False, None) + if not f: + self.handle.free(True) + self.handle = None + return False + self.cookie = self.calc_cookie_fp(f) + if self.usecachedrepo(None, True): + return True + self.handle.add_repomdxml(f) + fname, fchksum = self.find("primary") + if not fname: + return False + f = self.read_repo_metadata(fname, True, fchksum) + if not f: + return False + self.handle.add_rpmmd(f, None) + self.add_exts() + self.writecachedrepo(None) + # Must be called after writing the repo + self.handle.create_stubs() + return True + + def read_repo_metadata(self, fname, uncompress, chksum): + f = open("{}/{}".format(self.metadata_path, fname)) + return solv.xfopen_fd(fname if uncompress else None, f.fileno()) + + def find(self, what): + di = self.handle.Dataiterator_meta(solv.REPOSITORY_REPOMD_TYPE, what, solv.Dataiterator.SEARCH_STRING) + di.prepend_keyname(solv.REPOSITORY_REPOMD) + for d in di: + dp = d.parentpos() + filename = dp.lookup_str(solv.REPOSITORY_REPOMD_LOCATION) + chksum = dp.lookup_checksum(solv.REPOSITORY_REPOMD_CHECKSUM) + if filename: + if not chksum: + print("No {} file checksum!".format(filename)) + return filename, chksum + return None, None + + def add_ext_keys(self, ext, repodata, handle): + if ext == "FL": + repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.SOLVABLE_FILELIST) + repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.REPOKEY_TYPE_DIRSTRARRAY) + else: + raise NotImplementedError + + def add_ext(self, repodata, what, ext): + filename, chksum = self.find(what) + if not filename: + return + handle = repodata.new_handle() + repodata.set_poolstr(handle, solv.REPOSITORY_REPOMD_TYPE, what) + repodata.set_str(handle, solv.REPOSITORY_REPOMD_LOCATION, filename) + repodata.set_checksum(handle, solv.REPOSITORY_REPOMD_CHECKSUM, chksum) + self.add_ext_keys(ext, repodata, handle) + repodata.add_flexarray(solv.SOLVID_META, solv.REPOSITORY_EXTERNAL, handle) + + def add_exts(self): + repodata = self.handle.add_repodata() + self.add_ext(repodata, "filelists", "FL") + repodata.internalize() + + def load_ext(self, repodata): + repomdtype = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_TYPE) + if repomdtype == "filelists": + ext = "FL" + else: + assert False + if self.usecachedrepo(ext): + return True + filename = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_LOCATION) + filechksum = repodata.lookup_checksum(solv.SOLVID_META, solv.REPOSITORY_REPOMD_CHECKSUM) + f = self.read_repo_metadata(filename, True, filechksum) + if not f: + return False + if ext == "FL": + self.handle.add_rpmmd(f, "FL", solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES | solv.Repo.REPO_LOCALPOOL) + self.writecachedrepo(ext, repodata) + return True + + def updateaddedprovides(self, addedprovides): + if self.handle.isempty(): + return + # make sure there's just one real repodata with extensions + repodata = self.handle.first_repodata() + if not repodata: + return + oldaddedprovides = repodata.lookup_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES) + if not set(addedprovides) <= set(oldaddedprovides): + for id in addedprovides: + repodata.add_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES, id) + repodata.internalize() + self.writecachedrepo(None, repodata) + +def load_stub(repodata): + repo = repodata.repo.appdata + if repo: + return repo.load_ext(repodata) + return False + +def setup_repos(): + + srcrepo = Repo("f27-source", _SOURCE_PACKAGE_INFO.local_cache_path) + repo = Repo("f27", _x86_64_PACKAGE_INFO.local_cache_path) + repo.srcrepo = srcrepo + updates_srcrepo = Repo("f27-updates-source", _SOURCE_UPDATES_INFO.local_cache_path) + updates_repo = Repo("f27-updates", _x86_64_UPDATES_INFO.local_cache_path) + updates_repo.srcrepo = updates_srcrepo + return [repo, srcrepo, updates_repo, updates_srcrepo] diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py new file mode 100644 index 0000000..514a0f4 --- /dev/null +++ b/src/_fedmod/cli.py @@ -0,0 +1,72 @@ +import sys +import argparse +import logging + +from .module_generator import ModuleGenerator +from . import _depchase, _repodata + +# TODO: Switch this over to click (already a dependency for progress bars) + +class ModtoolsCLI(object): + """ Class for processing data from commandline """ + + @staticmethod + def build_parser(): + parser = argparse.ArgumentParser(description="Generates module related files") + base_parser = argparse.ArgumentParser(add_help=False) + base_parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="verbose operation" + ) + + subparsers = parser.add_subparsers(dest="cmd_name") + + parser_rpm2module = subparsers.add_parser( + 'rpm2module', parents=[base_parser], + help="Generates modulemd file", + description="Gets package info and dependencies and creates modulemd file." + ) + parser_rpm2module.add_argument( + "pkgs", + metavar='PKGS', + nargs='+', + help="Specify list of packages for module.", + ) + + parser_metadata = subparsers.add_parser( + 'fetch-metadata', parents=[base_parser], + help="Fetches repository metadata", + description="Caches needed repository metadata locally" + ) + + return parser + + def __init__(self, args=None): + self.parser = ModtoolsCLI.build_parser() + self.args = self.parser.parse_args(args) + + def __getattr__(self, name): + try: + return getattr(self.args, name) + except AttributeError: + return object.__getattribute__(self, name) + + +def run(): + try: + cli = ModtoolsCLI(sys.argv[1:]) + if cli.args.verbose: + logging.basicConfig(level=logging.INFO) + if cli.args.cmd_name == 'rpm2module': + mg = ModuleGenerator(cli.args.pkgs) + mg.run() + elif cli.args.cmd_name == 'fetch-metadata': + _repodata.download_repo_metadata() + + except KeyboardInterrupt: + print('\nInterrupted by user') + except Exception as e: + logging.exception("Unexpected exception") + sys.exit(1) diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py new file mode 100644 index 0000000..33836db --- /dev/null +++ b/src/_fedmod/module_generator.py @@ -0,0 +1,151 @@ +from __future__ import absolute_import + +import modulemd +import logging +import dnf +from . import _depchase + +def _name_only(rpm_name): + name, version, release = rpm_name.rsplit("-", 2) + return name + +def _categorise_deps(pool, all_rpm_deps): + module_deps = set() + remaining_rpm_deps = set() + for pkg in all_rpm_deps: + modname = _depchase.get_module_for_rpm(pkg.name) + if modname is not None: + module_deps.add(modname) + else: + remaining_rpm_deps.add(pkg) + return module_deps, remaining_rpm_deps + +class ModuleGenerator(object): + + def __init__(self, pkgs): + self.pkgs = pkgs + self.pkg = None + self.mmd = modulemd.ModuleMetadata() + self._calculate_dependencies() + + def _calculate_dependencies(self): + pkgs = self.pkgs + pool = _depchase.make_pool("x86_64") + self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} + run_deps = _depchase.ensure_installable(pkgs, pool=pool) + module_run_deps, rpm_run_deps = _categorise_deps(pool, run_deps) + self.module_run_deps = module_run_deps + run_srpms, build_deps = _depchase.ensure_buildable(rpm_run_deps) + module_build_deps = set() + resolved_build_deps = set() + build_srpms = set() + for i in range(10): + # Arbitrary bound of 10 levels of SRPM bootstrapping + new_module_build_deps, remaining_build_deps = _categorise_deps(pool, build_deps) + module_build_deps |= new_module_build_deps + resolved_build_deps |= (build_deps - remaining_build_deps) + build_deps -= resolved_build_deps + if build_deps: + new_build_srpms, remaining_build_deps = _depchase.ensure_buildable(build_deps) + build_srpms |= new_build_srpms + resolved_build_deps |= (build_deps - remaining_build_deps) + build_deps -= resolved_build_deps + if not build_deps: + break + else: + logging.warn("Failed to close out build dependencies after 10 iteration") + self.module_run_deps = module_run_deps + self.module_build_deps = module_build_deps + run_srpm_names = {_name_only(n) for n in run_srpms} + build_srpm_names = {_name_only(n) for n in build_srpms} + self.run_srpms = run_srpm_names - build_srpm_names + self.build_srpms = build_srpm_names - run_srpm_names + self.build_and_run_srpms = run_srpm_names & build_srpm_names + self.unresolved_build_rpms = {_name_only(s.name) for s in build_deps} + + + def _get_pkg_info(self): + """Function loads package from dnf""" + # TODO: Get this from the _depchase metadata, not the system metadata + logging.info("Getting package info from DNF") + b = dnf.Base() + b.read_all_repos() + b.fill_sack() + q = b.sack.query().filter(name=self.pkgs, reponame='fedora', latest=True) + if len(q) > 1: + raise ValueError('Name of package is not unique') + if len(q) == 0: + raise ValueError('No package found in repo') + self.pkg = q[0] + + def _save_module_md(self): + """ + Function saves modulemd file to the current directory + based on argument name + :return: + """ + + if len(self.pkgs) == 1: + file_name = self.pkgs[0] + '.yaml' + else: + file_name = "modulemd-output.yaml" + self.mmd.dump(file_name) + print('Modulemd file is generated here ./%s' % file_name) + return True + + def _update_module_md(self): + """ + Function updates modulemd file with dependencies + are information taken from SPEC file. + :return: + """ + self.mmd.add_module_license("MIT") + + if len(self.pkgs) == 1: + self.mmd.summary = str(self.pkg.summary) + self.mmd.description = str(self.pkg.description) + + # Default license for the module metadata, same as default Fedora + # content license. + + self.mmd.add_content_license(str(self.pkg.license)) + + # Declare the public API + for pkg in self.api_srpms: + self.mmd.api.add_rpm(pkg) + self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg)) + + # Declare module level dependencies + for modname in self.module_build_deps: + self.mmd.buildrequires[modname] = "f27" + for modname in self.module_run_deps: + self.mmd.requires[modname] = "f27" + + # Add any other RPMs not available from existing modules as components + for pkg in self.build_and_run_srpms: + self.mmd.components.add_rpm(pkg, "Build and runtime dependency.", buildorder=self._get_build_order(pkg)) + + for pkg in self.run_srpms: + self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg)) + + for pkg in self.build_srpms: + self.mmd.components.add_rpm(pkg, "Build dependency.", buildorder=self._get_build_order(pkg)) + # Filter out any build-only packages + # TODO: This won't filter out all the RPMs, only the one matching the SRPM name + # See https://pagure.io/modulemd/issue/54 for discussion + self.mmd.filter.add_rpm(pkg) + + # TODO: Emit something for non-empty self.unresolved_build_rpms + + + def _get_build_order(self, pkg): + if pkg in self.mmd.api.rpms: + return 10 + else: + return 0 + + def run(self): + if len(self.pkgs) == 1: + self._get_pkg_info() + self._update_module_md() + self._save_module_md() diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..447b4e7 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +from setuptools import setup, find_packages + +setup( + name='fedmod', + version='0.0.1', + author='Dominika Hodovska', + author_email='dhodovsk@redhat.com', + maintainer='Nick Coghlan', + maintainer_email='ncoghlan@redhat.com', + description='Utilities for generating & maintaining modulemd files', + long_description=( + "fedmod provides tools for converting existing RPMs (most notably " + "metapackages) into module definitions in Fedora's modulemd format." + ), + license='MIT', + keywords='modularization modularity module modulemd fedora', + url='https://pagure.io/modularity/fedmod', + entry_points={ + 'console_scripts': [ + 'fedmod=_fedmod.cli:run' + ] + }, + install_requires=[ + 'modulemd', + 'click', + 'requests', + 'requests-toolbelt', + 'lxml', + 'attrs', + ], + packages=find_packages(), +) diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index f12ae8a..e711b91 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -1,7 +1,7 @@ import pytest import os.path -from fedmod.cli import ModtoolsCLI -from fedmod.module_generator import ModuleGenerator +from _fedmod.cli import ModtoolsCLI +from _fedmod.module_generator import ModuleGenerator def _generate_modulemd(rpms): cmd_input = list(['rpm2module'])