From 1c31748dfe89ac5c362d1cc3578029415bdaada7 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Apr 23 2020 12:47:23 +0000 Subject: kojid: delete mergerepos script --- diff --git a/builder/Makefile b/builder/Makefile index 9d16298..c78cefe 100644 --- a/builder/Makefile +++ b/builder/Makefile @@ -1,5 +1,4 @@ BINFILES = kojid -LIBEXECFILES = mergerepos SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir) TYPE = systemd @@ -21,9 +20,6 @@ _install: mkdir -p $(DESTDIR)/usr/sbin install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin - mkdir -p $(DESTDIR)/usr/libexec/kojid - install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid - mkdir -p $(DESTDIR)/etc/mock/koji mkdir -p $(DESTDIR)/etc/kojid diff --git a/builder/kojid b/builder/kojid index a105d75..2d5754f 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5506,22 +5506,18 @@ class CreaterepoTask(BaseTaskHandler): ext_url = ext_url.replace('$arch', arch) repos.append(ext_url) + if not self.options.use_createrepo_c: + raise koji.GenericError("use_createrepo_c should always be enabled") + # construct command if merge_mode == 'simple': - # currently only supported by our own mergerepos script - # (we need it to write pkgorigins) - cmd = ['/usr/libexec/kojid/mergerepos', - '--mode', 'simple', - '--tempdir', self.workdir] + cmd = ['/usr/bin/mergerepo_c', '--koji', '--simple', '--pkgorigins'] elif merge_mode == 'bare': # "bare" merge mode for repos with modular metadata # forces use of mergerepo_c cmd = ['/usr/bin/mergerepo_c', '--pkgorigins', '--all'] - elif self.options.use_createrepo_c: - cmd = ['/usr/bin/mergerepo_c', '--koji'] else: - cmd = ['/usr/libexec/kojid/mergerepos'] - cmd.extend(['--tempdir', self.workdir]) + cmd = ['/usr/bin/mergerepo_c', '--koji'] if merge_mode != 'bare': blocklist = self.repodir + '/blocklist' cmd.extend(['-b', blocklist]) diff --git a/builder/mergerepos b/builder/mergerepos deleted file mode 100755 index a6c5a35..0000000 --- a/builder/mergerepos +++ /dev/null @@ -1,369 +0,0 @@ -#!/usr/bin/python2 - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -# Copyright 2009-2014 Red Hat, Inc. -# Written by Mike Bonnet - -# Merge repos using rules specific to Koji -# Largely borrowed from the mergerepo script included in createrepo and -# written by Seth Vidal - -from __future__ import absolute_import - -import os.path -import shutil -import sys -import tempfile -from optparse import OptionParser - -import createrepo -import rpmUtils.miscutils -import yum -import yum.misc - -# Expand a canonical arch to the full list of -# arches that should be included in the repo. -# Basically the inverse of koji.canonArch(). -# Lists taken from rpmUtils.arch. -EXPAND_ARCHES = { - 'i386': ['i486', 'i586', 'geode', 'i686', 'athlon'], - 'x86_64': ['ia32e', 'amd64'], - 'ppc64': ['ppc64pseries', 'ppc64iseries'], - 'sparc64': ['sparc64v', 'sparc64v2'], - 'sparc': ['sparcv8', 'sparcv9', 'sparcv9v', 'sparcv9v2'], - 'alpha': ['alphaev4', 'alphaev45', 'alphaev5', 'alphaev56', - 'alphapca56', 'alphaev6', 'alphaev67', 'alphaev68', 'alphaev7'], - 'armhfp': ['armv7hl', 'armv7hnl', 'armv6hl', 'armv6hnl'], - 'arm': ['armv5tel', 'armv5tejl', 'armv6l', 'armv7l'], - 'sh4': ['sh4a'] -} - -MULTILIB_ARCHES = { - 'x86_64': 'i386', - 'ppc64': 'ppc', - 's390x': 's390' -} - - -def parse_args(args): - """Parse our opts/args""" - usage = """ - mergerepos: take 2 or more repositories and merge their metadata into a new - repo using Koji semantics - - mergerepos --repo=url --repo=url --outputdir=/some/path""" - - parser = OptionParser(version="mergerepos 0.1", usage=usage) - # query options - parser.add_option("-r", "--repo", dest="repos", default=[], action="append", - help="repo url") - parser.add_option("-g", "--groupfile", default=None, - help="path to groupfile to include in metadata") - parser.add_option("-a", "--arch", dest="arches", default=[], action="append", - help="List of arches to include in the repo") - parser.add_option("-b", "--blocked", default=None, - help="A file containing a list of srpm names to exclude " - "from the merged repo") - parser.add_option("--mode", default='koji', help="Select the merge mode") - parser.add_option("-o", "--outputdir", default=None, - help="Location to create the repository") - parser.add_option("--tempdir", default=None, - help="Location for temporary files") - (opts, argsleft) = parser.parse_args(args) - - if len(opts.repos) < 1: - parser.print_usage() - sys.exit(1) - - # expand arches - for arch in opts.arches[:]: - if arch in EXPAND_ARCHES: - opts.arches.extend(EXPAND_ARCHES[arch]) - - # support multilib repos - for arch in opts.arches[:]: - multilib_arch = MULTILIB_ARCHES.get(arch) - if multilib_arch: - opts.arches.append(multilib_arch) - if multilib_arch in EXPAND_ARCHES: - opts.arches.extend(EXPAND_ARCHES[multilib_arch]) - - # always include noarch - if 'noarch' not in opts.arches: - opts.arches.append('noarch') - - if not opts.outputdir: - parser.error('You must specify an outputdir with -o') - sys.exit(1) - - return opts - - -def make_const_func(value): - """Return a function that returns the given value""" - return lambda *a: value - - -class RepoMerge(object): - def __init__(self, repolist, arches, groupfile, blocked, outputdir, - tempdir=None, mode='koji'): - self.repolist = repolist - self.outputdir = outputdir - self.tempdir = tempdir - self.mdconf = createrepo.MetaDataConfig() - # explicitly request sha1 for backward compatibility with older yum - self.mdconf.sumtype = 'sha1' - self.mdconf.database = True - self.mdconf.verbose = True - self.mdconf.changelog_limit = 3 - self.yumbase = yum.YumBase() - if hasattr(self.yumbase, 'preconf'): - self.yumbase.preconf.fn = '/dev/null' - self.yumbase.preconf.init_plugins = False - self.yumbase.preconf.debuglevel = 2 - else: - self.yumbase._getConfig('/dev/null', init_plugins=False, debuglevel=2) - self.yumbase.conf.cachedir = tempfile.mkdtemp(dir=self.tempdir) - self.yumbase.conf.cache = 0 - self.archlist = arches - self.mdconf.groupfile = groupfile - self.blocked = blocked - self.mode = mode - - def close(self): - if self.yumbase is not None: - cachedir = self.yumbase.conf.cachedir - self.yumbase.close() - self.yumbase = None - self.mdconf = None - if os.path.isdir(cachedir): - shutil.rmtree(cachedir) - - def __del__(self): - self.close() - - def merge_repos(self): - self.yumbase.repos.disableRepo('*') - # add our repos and give them a merge rank in the order they appear in - # in the repolist - count = 0 - for r in self.repolist: - count += 1 - rid = 'repo%s' % count - sys.stderr.write('Adding repo: %s\n' % r) - n = self.yumbase.add_enable_repo(rid, baseurls=[r]) - n._merge_rank = count - - # setup our sacks - self.yumbase._getSacks(archlist=self.archlist) - - self.sort_and_filter() - - def sort_and_filter(self): - """ - For each package object, check if the srpm name has ever been seen before. - If is has not, keep the package. If it has, check if the srpm name was first seen - in the same repo as the current package. If so, keep the package from the srpm with the - highest NVR. If not, keep the packages from the first srpm we found, and delete packages - from all other srpms. - - Packages with matching NVRs in multiple repos will be taken from the first repo. - - If the srpm name appears in the blocked package list, any packages generated from the srpm - will be deleted from the package sack as well. - - This method will also generate a file called "pkgorigins" and add it to the repo metadata. - This is a tab-separated map of package E:N-V-R.A to repo URL (as specified on the - command-line). This allows a package to be tracked back to its origin, even if the location - field in the repodata does not match the original repo location. - """ - - if self.mode == 'simple': - return self.do_simple_sort() - - # sort the repos by _merge_rank - # lowest number is the highest rank (1st place, 2nd place, etc.) - repos = self.yumbase.repos.listEnabled() - repos.sort(key=lambda o: o._merge_rank) - - include_srpms = {} - - # calculating what "builds" (srpms) we're allowing into the repo - seen_srpms = {} - for reponum, repo in enumerate(repos): - for pkg in repo.sack: - if reponum == 0 and not pkg.basepath: - # this is the first repo (i.e. the koji repo) and appears - # to be using relative urls - # XXX - kind of a hack, but yum leaves us little choice - # force the pkg object to report a relative location - loc = """\n""" % yum.misc.to_xml(pkg.remote_path, - attrib=True) - pkg._return_remote_location = make_const_func(loc) - if pkg.sourcerpm in seen_srpms: - # we're just looking at sourcerpms this pass and we've - # already seen this one - continue - seen_srpms[pkg.sourcerpm] = 1 - srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm) - if srpm_name in include_srpms: - other_srpm, other_repoid = include_srpms[srpm_name] - if pkg.repoid != other_repoid: - # We found a rpm built from an srpm with the same name in a previous repo. - # The previous repo takes precedence, so ignore the srpm found here. - sys.stderr.write('Package %s already provided by repo %s' - ' (at %s in repo %s)\n' - % (srpm_name, other_repoid, str(pkg), pkg.repoid)) - continue - else: - # We're in the same repo, so compare srpm NVRs - other_srpm_name, other_ver, other_rel, other_epoch, other_arch = \ - rpmUtils.miscutils.splitFilename(other_srpm) - cmp = rpmUtils.miscutils.compareEVR((epoch, ver, rel), - (other_epoch, other_ver, other_rel)) - if cmp > 0: - # The current package we're processing is from a newer srpm than the - # existing srpm in the dict, so update the dict - include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid) - sys.stderr.write('Replacing older source nvr: ' - '%s > %s\n' % (pkg.sourcerpm, other_srpm)) - elif cmp < 0: - sys.stderr.write('Ignoring older source nvr: ' - '%s < %s\n' % (pkg.sourcerpm, other_srpm)) - # otherwise same, so we already have it - elif srpm_name in self.blocked: - sys.stderr.write('Ignoring blocked package: %s\n\n' % - pkg.sourcerpm) - continue - else: - include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid) - - pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') - origins = open(pkgorigins, 'w') - - seen_rpms = {} - for repo in repos: - for pkg in repo.sack: - srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm) - incl_srpm, incl_repoid = include_srpms.get(srpm_name, (None, None)) - pkg_nvra = str(pkg) - if pkg_nvra in seen_rpms: - sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra) - elif incl_srpm is None: - sys.stderr.write('Excluding %s (%s is blocked)\n' - % (pkg_nvra, srpm_name)) - repo.sack.delPackage(pkg) - elif incl_srpm == pkg.sourcerpm: - origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0])) - seen_rpms[pkg_nvra] = 1 - else: - sys.stderr.write('Excluding %s (wrong srpm version ' - '%s != %s)\n' % (pkg_nvra, pkg.sourcerpm, incl_srpm)) - repo.sack.delPackage(pkg) - - origins.close() - self.mdconf.additional_metadata['origin'] = pkgorigins - - def do_simple_sort(self): - """ - Handle the 'sort_and_filter' case when mode=simple - - As the name implies, this is a much simpler approach. Mainly, we need - to generate the pkgorigins file. - """ - - # sort the repos by _merge_rank - # lowest number is the highest rank (1st place, 2nd place, etc.) - repos = self.yumbase.repos.listEnabled() - repos.sort(key=lambda o: o._merge_rank) - - # TODO: reduce duplication between this function and sort_and_filter() - - # We lack the complex filtration of mode=koji, but we still need to: - # - fix urls for primary repo - # - enforce blocked list - for reponum, repo in enumerate(repos): - for pkg in repo.sack: - if reponum == 0 and not pkg.basepath: - # this is the first repo (i.e. the koji repo) and appears - # to be using relative urls - # XXX - kind of a hack, but yum leaves us little choice - # force the pkg object to report a relative location - loc = """\n""" % yum.misc.to_xml(pkg.remote_path, - attrib=True) - pkg._return_remote_location = make_const_func(loc) - - pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') - origins = open(pkgorigins, 'w') - - seen_rpms = {} - for repo in repos: - for pkg in repo.sack: - srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm) - pkg_nvra = str(pkg) - if pkg_nvra in seen_rpms: - sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra) - # note: we warn, but do not omit it - if srpm_name in self.blocked: - sys.stderr.write('Ignoring blocked package: %s\n\n' % - pkg.sourcerpm) - repo.sack.delPackage(pkg) - if pkg_nvra not in seen_rpms: - origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0])) - seen_rpms[pkg_nvra] = 1 - origins.close() - self.mdconf.additional_metadata['origin'] = pkgorigins - - def write_metadata(self): - self.mdconf.pkglist = self.yumbase.pkgSack - self.mdconf.directory = self.outputdir - # clean out what was there - if os.path.exists(self.mdconf.directory + '/repodata'): - shutil.rmtree(self.mdconf.directory + '/repodata') - - if not os.path.exists(self.mdconf.directory): - os.makedirs(self.mdconf.directory) - - mdgen = createrepo.MetaDataGenerator(config_obj=self.mdconf) - mdgen.doPkgMetadata() - mdgen.doRepoMetadata() - mdgen.doFinalMove() - - -def main(args): - """main""" - opts = parse_args(args) - - if opts.blocked: - blocked_fo = open(opts.blocked) - blocked_list = blocked_fo.readlines() - blocked_fo.close() - blocked = dict([(b.strip(), 1) for b in blocked_list]) - else: - blocked = {} - - merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked, - opts.outputdir, opts.tempdir, opts.mode) - - try: - merge.merge_repos() - merge.write_metadata() - finally: - merge.close() - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/koji.spec b/koji.spec index 07418da..e3b0fa7 100644 --- a/koji.spec +++ b/koji.spec @@ -317,6 +317,9 @@ Requires: /usr/bin/cvs Requires: /usr/bin/svn Requires: /usr/bin/git Requires: createrepo >= 0.9.2 +%if 0%{?fedora} || 0%{?rhel} >= 8 +Recommends: createrepo_c >= 0.15.1 +%endif %if 0%{py3_support} > 1 Requires: python%{python3_pkgversion}-%{name} = %{version}-%{release} Requires: python%{python3_pkgversion}-librepo @@ -627,8 +630,6 @@ rm -rf $RPM_BUILD_ROOT %files builder %{_sbindir}/kojid -%dir %{_libexecdir}/kojid -%{_libexecdir}/kojid/mergerepos %if %{use_systemd} %{_unitdir}/kojid.service %else