From 146f8297853dc59dc8c6e3385e539fa653f084d9 Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 29 2017 17:53:56 +0000 Subject: [PATCH 1/5] depchase: Fix get_rpms_for_srpms lookup mapping The logic here is flawed, adding the arbitrarily first returned package name from libsolv to the sources list, when we should be adding the package name we actually compared against. The one caller is used for populating the RPM list for the bootstrap module: previously the code thought there was only around 1700 RPMs in bootstrap, after this fix it's closer to 12000 --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 9f2d340..ab66100 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -215,7 +215,6 @@ def get_srpm_for_rpm(pool, pkg): return get_sourcepkg(s, only_name=True) def get_rpms_for_srpms(pool, pkgnames): - sources = set() for n in pkgnames: sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE) @@ -223,7 +222,7 @@ def get_rpms_for_srpms(pool, pkgnames): found = sel.solvables() for rpm in found: if rpm.arch in ("src", "nosrc"): - sources.add(str(found[0])) + sources.add(str(rpm)) break else: # Multilib means we may see multiple binary RPMs with the same name From 776ef692e5e2b86019edd540834508b7ea5a9660 Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 29 2017 17:53:56 +0000 Subject: [PATCH 2/5] depchase: Drop unnecessary archful lookup for get_rpms_for_srpms This code path isn't used. The only input to this function comes from modulemd 'components' which are always SRPMs, and below in the code we are comparing against get_sourcepkg output, which is always an SRPM. --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index ab66100..8899225 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -217,16 +217,10 @@ def get_srpm_for_rpm(pool, pkg): def get_rpms_for_srpms(pool, pkgnames): sources = set() for n in pkgnames: - sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE) + sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_SOURCE_ONLY) if not sel.isempty(): - found = sel.solvables() - for rpm in found: - if rpm.arch in ("src", "nosrc"): - sources.add(str(rpm)) - break - else: - # Multilib means we may see multiple binary RPMs with the same name - sources.add(get_sourcepkg(rpm, only_name=True)) + srcrpm = sel.solvables()[0] + sources.add(str(srcrpm)) # This search is O(N) where N = the number of packages in Fedora # so it would be nice to find a more algorithmically efficient approach From 4c4483d69c453163a195773ca973c97fb66bf0ae Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 29 2017 17:53:56 +0000 Subject: [PATCH 3/5] depchase: Handle epoch packages in get_rpms_for_srpms Currently we add str(solvable) to the sources list, but compare against edited solvable.lookup_sourcepkg(). The former is an RPM NVR, the latter is a filename. This works for most cases, unless the package has an Epoch, which is represented in the former, but not the latter. To properly compare against NVR requires running another libsolv query to lookup the package associated with the lookup_sourcepkg() output. Instead, grab the filename like output from the SRPM solv object up front. This detects another 5000 packages as correctly belonging to the bootstrap module --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 8899225..4adeea7 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -220,13 +220,16 @@ def get_rpms_for_srpms(pool, pkgnames): sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_SOURCE_ONLY) if not sel.isempty(): srcrpm = sel.solvables()[0] - sources.add(str(srcrpm)) + # lookup_location() here will be something like + # Packages/n/nginx-1.12.1-1.fc27.src.rpm + # Which is closer to lookup_sourcepkg() which we match with below + sources.add(os.path.basename(srcrpm.lookup_location()[0])) # This search is O(N) where N = the number of packages in Fedora # so it would be nice to find a more algorithmically efficient approach # OTOH, we only run this when *fetching* metadata, so it isn't too bad result = set() for p in (s for s in pool.solvables if s.arch not in ("src", "nosrc")): - if get_sourcepkg(p, only_name=True) in sources: + if p.lookup_sourcepkg() in sources: result.add(p.name) return result From ef8166840e1c05c42cb9110c44add13f5065f311 Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 29 2017 18:00:56 +0000 Subject: [PATCH 4/5] cli: add rpms-from-srpm command Returns all RPMs that are generated from the passed SRPM name. This is useful when packaging new modules. For example, I see glusterfs needs to bundle pyxattr since no other module currently provides it. It's handy to have a simple place to see how many sub-RPMs pyxattr builds to get an idea of what might need to be filtered. --- diff --git a/src/README.md b/src/README.md index a1fd675..3140691 100644 --- a/src/README.md +++ b/src/README.md @@ -90,6 +90,19 @@ pkg1-0:2.4.28-3.module_e7ab08d3.x86_64 pkg2-0:4.5.20-1.module_e7ab08d3.x86_64 ``` +**Lookup RPMs from SRPM** + +Lists all the RPMs generated by the specified SRPM package name. + +``` +$ ./fedmod rpms-from-srpm spkg +pkg1 +pkg2 +pkg3 +... +``` + + ## Modulemd creation Before generating any draft modulemd files, first run the following command to diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index b5f48cb..93af2df 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -118,3 +118,12 @@ def which_srpm(pkg): """Reports which SRPM produces the given RPM""" rq = ModuleRepoquery() rq.get_srpm_for_rpm(pkg) + +# Map SRPM to the RPMs it generates. Useful info when building +# new modules, choosing the API or what to filter +@_cli_commands.command('rpms-from-srpm') +@click.argument("pkg") +def rpms_from_srpm(pkg): + """Reports which RPMS are generated from the given SRPM""" + rq = ModuleRepoquery() + rq.get_rpms_for_srpm(pkg) diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index be7d5bf..93d3f1f 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -68,5 +68,8 @@ class ModuleRepoquery(object): if srpm: print(_name_only(srpm)) - - \ No newline at end of file + def get_rpms_for_srpm(self, pkg): + pool = _depchase.make_pool("x86_64") + rpms = _depchase.get_rpms_for_srpms(pool, [pkg]) + for rpm in sorted(rpms): + print(rpm) diff --git a/tests/test_module_repoquery.py b/tests/test_module_repoquery.py index 9ee25d5..b8b355a 100644 --- a/tests/test_module_repoquery.py +++ b/tests/test_module_repoquery.py @@ -92,3 +92,16 @@ class TestGettingSRPMOfRPM(object): out, err = capfd.readouterr() assert "mailcap" in out + + +class TestGettingRPMSFromSRPM(object): + + def setup(self): + self.mr = ModuleRepoquery() + + def test_get_rpms_for_rpm(self, capfd): + self.mr.get_rpms_for_srpm("nginx") + out, err = capfd.readouterr() + + assert "nginx-all-modules" in out + assert "nginx-mod-http-perl" in out From e7ad4c27dbabe64cd9ad73d989f2a0fc4284448c Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 29 2017 18:01:47 +0000 Subject: [PATCH 5/5] README: Document srpm-of-rpm command --- diff --git a/src/README.md b/src/README.md index 3140691..c8f842e 100644 --- a/src/README.md +++ b/src/README.md @@ -102,6 +102,15 @@ pkg3 ... ``` +**Lookup SRPM of RPM** + +Return the SRPM package name that generates the passed RPM + +``` +$ ./fedmod srpm-of-rpm pkg1 +spkg +``` + ## Modulemd creation