From bb9dd3009b5fcf1bbffe029fed59f354b8f643fb Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Oct 05 2017 08:17:20 +0000 Subject: Start migration to depchase - add depchase as a private module - remove mbs-build cache dependency --- diff --git a/README.md b/README.md index 5f01076..ed292b8 100644 --- a/README.md +++ b/README.md @@ -61,19 +61,6 @@ The tests can then be run in the launched subshell with: Note: the tests will currently fail. Fixing that is a work in progress :) -### Additional preparation of the local mbs-build cache - -Running the tests currently also depends on having previously built at least -one module locally with `mbs-build`, as described in the [local module build -instructions](https://docs.pagure.org/modularity/development/building-modules/building-local.html). - - $ fedpkg clone --branch f26 modules/haproxy - $ cd haproxy - $ mbs-build local - -Note: These commands can be run in any directory, they don't have to be run -from the `fedmod` source checkout. - ### Reviewing dependencies To see the Python level dependencies graph: diff --git a/fedmod/_depchase.py b/fedmod/_depchase.py new file mode 100644 index 0000000..fd140a9 --- /dev/null +++ b/fedmod/_depchase.py @@ -0,0 +1,498 @@ +#!/usr/bin/python3 +import configparser +import itertools +import logging +import os +import sys +import tempfile +import click +import smartcols +import solv + +XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") +CACHEDIR = os.path.join(XDG_CACHE_HOME, "depchase") + +logger = logging.getLogger("depchase") + +class Repo(object): + def __init__(self, name, baseurl): + self.name = name + self.baseurl = baseurl + self.handle = None + self.cookie = None + self.extcookie = None + self.srcrepo = None + + @staticmethod + def calc_cookie_fp(fp): + chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) + chksum.add("1.1") + chksum.add_fp(fp) + return chksum.raw() + + @staticmethod + def calc_cookie_ext(f, cookie): + chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) + chksum.add("1.1") + chksum.add(cookie) + chksum.add_fstat(f.fileno()) + return chksum.raw() + + def cachepath(self, ext=None): + path = "{}-{}".format(self.name.replace(".", "_"), self.baseurl) + if ext: + path = "{}-{}.solvx".format(path, ext) + else: + path = "{}.solv".format(path) + return os.path.join(CACHEDIR, path.replace("/", "_")) + + def usecachedrepo(self, ext, mark=False): + try: + repopath = self.cachepath(ext) + f = open(repopath, "rb") + f.seek(-32, os.SEEK_END) + fcookie = f.read(32) + if len(fcookie) != 32: + return False + cookie = self.extcookie if ext else self.cookie + if cookie and fcookie != cookie: + return False + if not ext: + f.seek(-32 * 2, os.SEEK_END) + fextcookie = f.read(32) + if len(fextcookie) != 32: + return False + f.seek(0) + f = solv.xfopen_fd(None, f.fileno()) + flags = 0 + if ext: + flags = solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES + if ext != "DL": + flags |= solv.Repo.REPO_LOCALPOOL + if not self.handle.add_solv(f, flags): + return False + if not ext: + self.cookie = fcookie + self.extcookie = fextcookie + if mark: + # no futimes in python? + try: + os.utime(repopath, None) + except Exception: + pass + except IOError: + return False + return True + + def writecachedrepo(self, ext, repodata=None): + tmpname = None + try: + if not os.path.isdir(CACHEDIR): + os.mkdir(CACHEDIR, 0o755) + fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", dir=CACHEDIR) + os.fchmod(fd, 0o444) + f = os.fdopen(fd, "wb+") + f = solv.xfopen_fd(None, f.fileno()) + if not repodata: + self.handle.write(f) + elif ext: + repodata.write(f) + else: + # rewrite_repos case, do not write stubs + self.handle.write_first_repodata(f) + f.flush() + if not ext: + if not self.extcookie: + self.extcookie = self.calc_cookie_ext(f, self.cookie) + f.write(self.extcookie) + if not ext: + f.write(self.cookie) + else: + f.write(self.extcookie) + f.close + if self.handle.iscontiguous(): + # switch to saved repo to activate paging and save memory + nf = solv.xfopen(tmpname) + if not ext: + # main repo + self.handle.empty() + flags = solv.Repo.SOLV_ADD_NO_STUBS + if repodata: + # rewrite repos case, recreate stubs + flags = 0 + if not self.handle.add_solv(nf, flags): + sys.exit("internal error, cannot reload solv file") + else: + # extension repodata + # need to extend to repo boundaries, as this is how + # repodata.write() has written the data + repodata.extend_to_repo() + flags = solv.Repo.REPO_EXTEND_SOLVABLES + if ext != "DL": + flags |= solv.Repo.REPO_LOCALPOOL + repodata.add_solv(nf, flags) + os.rename(tmpname, self.cachepath(ext)) + except (OSError, IOError): + if tmpname: + os.unlink(tmpname) + + def load(self, pool): + assert not self.handle + self.handle = pool.add_repo(self.name) + self.handle.appdata = self + f = self.download("repodata/repomd.xml", False, None) + if not f: + self.handle.free(True) + self.handle = None + return False + self.cookie = self.calc_cookie_fp(f) + if self.usecachedrepo(None, True): + return True + self.handle.add_repomdxml(f) + fname, fchksum = self.find("primary") + if not fname: + return False + f = self.download(fname, True, fchksum) + if not f: + return False + self.handle.add_rpmmd(f, None) + self.add_exts() + self.writecachedrepo(None) + # Must be called after writing the repo + self.handle.create_stubs() + return True + + def download(self, fname, uncompress, chksum): + f = open("{}/{}".format(self.baseurl, fname)) + return solv.xfopen_fd(fname if uncompress else None, f.fileno()) + + def find(self, what): + di = self.handle.Dataiterator_meta(solv.REPOSITORY_REPOMD_TYPE, what, solv.Dataiterator.SEARCH_STRING) + di.prepend_keyname(solv.REPOSITORY_REPOMD) + for d in di: + dp = d.parentpos() + filename = dp.lookup_str(solv.REPOSITORY_REPOMD_LOCATION) + chksum = dp.lookup_checksum(solv.REPOSITORY_REPOMD_CHECKSUM) + if filename: + if not chksum: + print("No {} file checksum!".format(filename)) + return filename, chksum + return None, None + + def add_ext_keys(self, ext, repodata, handle): + if ext == "FL": + repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.SOLVABLE_FILELIST) + repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.REPOKEY_TYPE_DIRSTRARRAY) + else: + raise NotImplementedError + + def add_ext(self, repodata, what, ext): + filename, chksum = self.find(what) + if not filename: + return + handle = repodata.new_handle() + repodata.set_poolstr(handle, solv.REPOSITORY_REPOMD_TYPE, what) + repodata.set_str(handle, solv.REPOSITORY_REPOMD_LOCATION, filename) + repodata.set_checksum(handle, solv.REPOSITORY_REPOMD_CHECKSUM, chksum) + self.add_ext_keys(ext, repodata, handle) + repodata.add_flexarray(solv.SOLVID_META, solv.REPOSITORY_EXTERNAL, handle) + + def add_exts(self): + repodata = self.handle.add_repodata() + self.add_ext(repodata, "filelists", "FL") + repodata.internalize() + + def load_ext(self, repodata): + repomdtype = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_TYPE) + if repomdtype == "filelists": + ext = "FL" + else: + assert False + if self.usecachedrepo(ext): + return True + filename = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_LOCATION) + filechksum = repodata.lookup_checksum(solv.SOLVID_META, solv.REPOSITORY_REPOMD_CHECKSUM) + f = self.download(filename, True, filechksum) + if not f: + return False + if ext == "FL": + self.handle.add_rpmmd(f, "FL", solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES | solv.Repo.REPO_LOCALPOOL) + self.writecachedrepo(ext, repodata) + return True + + def updateaddedprovides(self, addedprovides): + if self.handle.isempty(): + return + # make sure there's just one real repodata with extensions + repodata = self.handle.first_repodata() + if not repodata: + return + oldaddedprovides = repodata.lookup_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES) + if not set(addedprovides) <= set(oldaddedprovides): + for id in addedprovides: + repodata.add_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES, id) + repodata.internalize() + self.writecachedrepo(None, repodata) + +def load_stub(repodata): + repo = repodata.repo.appdata + if repo: + return repo.load_ext(repodata) + return False + +def setup_repos(conffile): + conf = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation()) + + with open(conffile, "r") as cfg: + conf.read_file(cfg) + + repos = {} + for sect in conf.sections(): + repos[sect] = Repo(sect, conf[sect]["path"]) + for repo in repos.values(): + if repo.name.endswith("-source"): + continue + repo.srcrepo = repos.get("{}-source".format(repo.name)) + if repo.srcrepo is None: + raise RuntimeError("{}-source repo is not defined".format(repo.name)) + return list(repos.values()) + +def setup_pool(arch, repos=()): + pool = solv.Pool() + #pool.set_debuglevel(2) + pool.setarch(arch) + pool.set_loadcallback(load_stub) + + for repo in repos: + repo.baseurl = repo.baseurl.format(arch=arch) + + for repo in repos: + assert repo.load(pool) + if "override" in repo.name: + repo.handle.priority = 99 + + addedprovides = pool.addfileprovides_queue() + if addedprovides: + for repo in repos: + repo.updateaddedprovides(addedprovides) + + pool.createwhatprovides() + + return pool + +def fix_deps(pool): + to_fix = ( + # Weak libcrypt-nss deps due to https://github.com/openSUSE/libsolv/issues/205 + ("glibc", solv.Selection.SELECTION_NAME, + solv.SOLVABLE_RECOMMENDS, lambda s: s.startswith("libcrypt-nss"), solv.SOLVABLE_SUGGESTS), + # Shim is not buildable + ("shim", solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_WITH_SOURCE, + solv.SOLVABLE_REQUIRES, lambda s: s in ("gnu-efi = 3.0w", "gnu-efi-devel = 3.0w"), None), + ) + for txt, flags, before, func, after in to_fix: + for s in pool.select(txt, flags).solvables(): + deps = s.lookup_deparray(before) + fixing = [dep for dep in deps if func(str(dep))] + for dep in fixing: + deps.remove(dep) + if after is not None: + s.add_deparray(after, dep) + # Use s.set_deparray() once will be available + s.unset(before) + for dep in deps: + s.add_deparray(before, dep) + +def get_sourcepkg(p, s=None, only_name=False): + if s is None: + s = p.lookup_sourcepkg()[:-4] + if only_name: + return s + # Let's try to find corresponding source + sel = p.pool.select(s, solv.Selection.SELECTION_CANON | solv.Selection.SELECTION_SOURCE_ONLY) + sel.filter(p.repo.appdata.srcrepo.handle.Selection()) + assert not sel.isempty(), "Could not find source package for {}".format(s) + solvables = sel.solvables() + assert len(solvables) == 1 + return solvables[0] + +def print_transaction(pool, transaction): + candq = transaction.newpackages() + if logger.getEffectiveLevel() <= logging.INFO: + tb = smartcols.Table() + tb.title = "DEPENDENCY INFORMATION" + cl = tb.new_column("INFO") + cl.tree = True + cl_match = tb.new_column("MATCH") + for p in candq: + ln = tb.new_line() + ln[cl] = str(p) + for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES): + lns = tb.new_line(ln) + lns[cl] = str(dep) + matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep)) + if not matches and str(dep).startswith("/"): + # Append provides by files + # TODO: use Dataiterator for getting filelist + matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq) + # It was possible to resolve set, so something is wrong here + assert matches + first = True + for m in matches: + if first: + lnc = lns + else: + lnss = tb.new_line(lns) + lnc = lnss + first = False + lnc[cl_match] = str(m) + logger.info(tb) + +def solve(solver, pkgnames, selfhost=False): + pool = solver.pool + + # We have to =( + fix_deps(pool) + + jobs = [] + # Initial jobs, no conflicting packages + for n in pkgnames: + sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) + assert not sel.isempty(), "Could not find package for {}".format(n) + jobs += sel.jobs(solv.Job.SOLVER_INSTALL) + problems = solver.solve(jobs) + if problems: + for problem in problems: + print(problem) + sys.exit(1) + + print_transaction(pool, solver.transaction()) + candq = [s for s in solver.transaction().newpackages() if s.arch not in ("src", "nosrc")] + sources = set(get_sourcepkg(s) for s in candq) + + if not selfhost: + return set(candq), sources + + # We already solved runtime requires, no need to do that twice + selfhosting = set(candq) + selfhosting_srcs = set() + candq = list(sources) + # We will store text-based view of processed srcs for better performance, + # because selections are not free + srcs_done = set() + while candq: + jobs = [pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE | solv.Job.SOLVER_WEAK, p.id) for p in candq] + solver.solve(jobs) + print_transaction(pool, solver.transaction()) + # We are interested to operate only on really new packages below + newpkgs = set(solver.transaction().newpackages()) - selfhosting + for p in newpkgs.copy(): + if p.arch in ("src", "nosrc"): + srcs_done.add(str(p)) + selfhosting_srcs.add(p) + newpkgs.remove(p) + continue + selfhosting |= newpkgs + + # SOLVER_FAVOR packages which we already solved which will help us to get small dependency chain + pool.setpooljobs(pool.getpooljobs() + [pool.Job(solv.Job.SOLVER_FAVOR | solv.Job.SOLVER_SOLVABLE, p.id) for p in newpkgs]) + + # In new queue only non-solvables are left + raw_decisions = solver.raw_decisions(1) + if not raw_decisions: + # At this point, nothing can be resolved anymore, so let's show problems + for p in candq: + job = pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE, p.id) + problems = solver.solve([job]) + # In some cases, even solvable jobs are disabled + # https://github.com/openSUSE/libsolv/issues/204 + #assert not problems + for problem in problems: + print(problem) + sys.exit(1) + candq = [s for s in candq if s.id not in raw_decisions] + + srcs_queued = set(str(p) for p in candq if p.arch in ("src", "nosrc")) + for p in newpkgs: + s = get_sourcepkg(p, only_name=True) + if s in srcs_done or s in srcs_queued: + continue + src = get_sourcepkg(p, s) + srcs_queued.add(str(src)) + candq.append(src) + + return selfhosting, selfhosting_srcs + + +def make_pool(arch, config): + return setup_pool(arch, setup_repos(config)) + +''' +@click.option("--recommends/--no-recommends", default=False, + help="Do not process optional (aka weak) dependencies.") +@click.option("--hint", multiple=True, + help=""" +Specify a package to have higher priority when more than one package could +satisfy a dependency. This option may be specified multiple times. + +For example, it is recommended to use --hint=glibc-minimal-langpack. +""") +@click.option("--selfhost", is_flag=True, + help="Look up the build dependencies as well.") +''' +def resolve(pool, pkgnames, recommends, hint, selfhost): + + # Set up initial hints + favorq = [] + for n in hint: + sel = pool.select(n, solv.Selection.SELECTION_NAME) + favorq += sel.jobs(solv.Job.SOLVER_FAVOR) + pool.setpooljobs(favorq) + + solver = pool.Solver() + if not recommends: + # Ignore weak deps + solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1) + + binary, source = solve(solver, pkgnames, selfhost=selfhost) + for p in itertools.chain(binary, source or ()): + print(p) + +def print_reldeps(pool, pkg): + sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) + assert not sel.isempty(), "Package can't be found" + found = sel.solvables() + assert len(found) == 1, "More matching solvables were found, {}".format(found) + s = found[0] + + reldep2str = {solv.SOLVABLE_REQUIRES: "requires", + solv.SOLVABLE_RECOMMENDS: "recommends", + solv.SOLVABLE_SUGGESTS: "suggests", + solv.SOLVABLE_SUPPLEMENTS: "supplements", + solv.SOLVABLE_ENHANCES: "enhances"} + for reltype, relstr in reldep2str.items(): + for dep in s.lookup_deparray(reltype): + print("{}: {}".format(relstr, dep)) + +def print_sourcepkg(pool, pkg): + sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) + assert not sel.isempty(), "Package can't be found" + found = sel.solvables() + assert len(found) == 1, "More matching solvables were found, {}".format(found) + s = found[0] + + print(get_sourcepkg(s, only_name=True)) + +def pkgs_by_sourcepkg(pool, sourcepkg): + + sel = pool.select(sourcepkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE) + if not sel.isempty(): + found = sel.solvables() + assert len(found) == 1, "More matching solvables were found, {}".format(found) + if found[0].arch in ("src", "nosrc"): + sourcepkg = str(found[0]) + else: + sourcepkg = get_sourcepkg(found[0], only_name=True) + + for p in (s for s in pool.solvables if s.arch not in ("src", "nosrc")): + if get_sourcepkg(p, only_name=True) == sourcepkg: + print(p) diff --git a/fedmod/module_deps_differ.py b/fedmod/module_deps_differ.py index b2cbc58..3b7e4da 100755 --- a/fedmod/module_deps_differ.py +++ b/fedmod/module_deps_differ.py @@ -1,5 +1,7 @@ import os import logging +import requests +from six.moves import configparser import dnf @@ -13,12 +15,20 @@ REPO_F27 = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everyth REPO_F27_SOURCE = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/source/tree/" ARCH = 'x86_64' + # TODO: add ability to solve more packages in one run # TODO: solve architecture! - as option? p = None +def download_repo_metadata(config_path, metadata_dir): + """TODO: Automate retrieving the required repo metadata""" + raise NotImplementedError( + "See https://github.com/fedora-modularity/depchase#installation " + "for download instructions" + ) + def get_pkgs_source_rpm_name(pkg): """ @@ -63,7 +73,7 @@ class ModuleDepsDiffer(object): return self.module_id_to_name.get(koji_tag_name, koji_tag_name) def obtain_module_names(self): - logging.info('Scanning built modules, this might take a while.') + logging.info('Getting module list from PDC, this might take a while.') j = self.p['unreleasedvariants']( variant_type="module", active=True, @@ -90,29 +100,8 @@ class ModuleDepsDiffer(object): return base def _init_repo_bases(self): - """ - version 1 is no longer usable since koji no longer creates repos for - modules this version iterates over repos present in - ~/modulebuild/cache/koji_tags, this means that you should do `mbs-build - local` to populate the directory - - Once we have real compose of boltron, we might utilize it here - - initialize repo objects for dnf to kick off queries - """ - logging.warn("Getting module information from mbs cache." + - " Please run mbs-build local with dependency modules" + - " you are interested in before running this script.") - cache_path = os.path.expanduser("~/modulebuild/cache/koji_tags") - for module_name in os.listdir(cache_path): - # for now ignore all what is not in f27 or rawhide stream, remove when situation changes - if not self.module_id_to_name[module_name].endswith((':f27', ':rawhide')): - continue - if module_name != "f27-modularity": # bootstrap contains almost everything, so let's ignore it - self.repos[module_name] = "file://{}".format(os.path.join(cache_path, module_name)) - for reponame, repourl in self.repos.items(): - base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl) - self.repo_bases[reponame] = base + # TODO: download the repo metadata containing module info + pass def get_package_requires(self): # runtime