From efd5277899236b5a0e1aa32e1f926a88aeb044e8 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Nov 24 2017 07:43:13 +0000 Subject: Issue #57: Cache metadata lookup tables Rather than deriving the required lookup tables from the repo metadata every time fedmod runs, instead run the derivation code as part of the fetch-metadata command, and cache the result as JSON files. There are still lots of opportunities for start-up time optimisations here, this is mainly aimed at separating fedmod's notion of a "working data set" from the raw repo level metadata that DNF uses. --- diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index 857e73c..219eea5 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -9,10 +9,11 @@ import logging import modulemd import solv import json -from attr import attributes, attrib -from requests_toolbelt.downloadutils.tee import tee_to_file +from collections import defaultdict from fnmatch import fnmatch from urllib.parse import urljoin +from attr import attributes, attrib +from requests_toolbelt.downloadutils.tee import tee_to_file from lxml import etree XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") @@ -73,7 +74,13 @@ _ALL_REPOS = ( _SOURCE_UPDATES_INFO, ) _BOOTSTRAP_MODULEMD = os.path.join(CACHEDIR, "f27-bootstrap.yaml") -_BOOTSTRAP_REVERSE_LOOKUP_CACHE = os.path.join(CACHEDIR, "f27-bootstrap-cache.json") + +_LOOKUP_CACHES = { + "_BOOTSTRAP_COMPONENTS_CACHE": os.path.join(CACHEDIR, "f27-bootstrap-cache.json"), + "_MODULE_FORWARD_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-module-contents-cache.json"), + "_SRPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-srpm-to-module-cache.json"), + "_RPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-rpm-to-module-cache.json"), +} METADATA_SECTIONS = ("filelists", "primary", "modules") @@ -133,6 +140,19 @@ def _download_metadata_files(repo_paths): # TODO: Actually prune old metadata files pass +def _write_cache(cache_name, data): + """Write the given data to the nominated cache file""" + cache_fname = _LOOKUP_CACHES[cache_name] + with open(cache_fname, "w") as cache_file: + json.dump(data, cache_file) + print(f" Added {cache_fname} to cache") + +def _read_cache(cache_name): + """Read the parsed data from the nominated cache file""" + cache_fname = _LOOKUP_CACHES[cache_name] + with open(cache_fname, "r") as cache_file: + return json.load(cache_file) + def _download_bootstrap_modulemd(): from ._depchase import make_pool, get_rpms_for_srpms print("Downloading build bootstrap module details") @@ -142,74 +162,79 @@ def _download_bootstrap_modulemd(): mmd = modulemd.ModuleMetadata() mmd.load(_BOOTSTRAP_MODULEMD) pool = make_pool("x86_64") - bootstrap_rpms = {} + bootstrap_rpms = set() rpms = get_rpms_for_srpms(pool, mmd.components.rpms) for rpmname in rpms: - bootstrap_rpms[rpmname] = "bootstrap" + bootstrap_rpms.add(rpmname) for srpmname in mmd.components.rpms: - bootstrap_rpms[srpmname] = "bootstrap" - with open(_BOOTSTRAP_REVERSE_LOOKUP_CACHE, "w") as cachefile: - json.dump(bootstrap_rpms, cachefile) - print(f" Added {_BOOTSTRAP_REVERSE_LOOKUP_CACHE} to cache") + bootstrap_rpms.add(srpmname) + _write_cache("_BOOTSTRAP_COMPONENTS_CACHE", list(bootstrap_rpms)) + +def _write_lookup_caches(): + metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) + repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") + repomd_xml = etree.parse(repomd_fname) + repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") + repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) + with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: + modules_yaml = modules_yaml_gz.read() + modules = modulemd.loads_all(modules_yaml) + module_forward_lookup = {} + srpm_reverse_lookup = defaultdict(list) + rpm_reverse_lookup = defaultdict(list) + for module in modules: + module_forward_lookup[module.name] = list(set(module.artifacts.rpms)) + for srpmname in module.components.rpms: + srpm_reverse_lookup[srpmname].append(module.name) + for rpmname in module.artifacts.rpms: + rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] + rpm_reverse_lookup[rpmprefix].append(module.name) + # Cache the lookup tables as local JSON files + print("Caching lookup tables") + _write_cache("_MODULE_FORWARD_LOOKUP_CACHE", module_forward_lookup) + _write_cache("_SRPM_REVERSE_LOOKUP_CACHE", srpm_reverse_lookup) + _write_cache("_RPM_REVERSE_LOOKUP_CACHE", rpm_reverse_lookup) + def download_repo_metadata(): """Downloads the latest repo metadata""" for repo_definition in _ALL_REPOS: _download_metadata_files(repo_definition) _download_bootstrap_modulemd() + _write_lookup_caches() + _SRPM_REVERSE_LOOKUP = {} # SRPM name : [module names] _RPM_REVERSE_LOOKUP = {} # RPM name : [module names] _BOOTSTRAP_COMPONENTS = set() _MODULE_FORWARD_LOOKUP = {} def _populate_module_reverse_lookup(): - # TODO: Construct and cache the reverse mapping as a JSON file as part of - # download_repo_metadata(), as with _BOOTSTRAP_REVERSE_LOOKUP_CACHE if _RPM_REVERSE_LOOKUP: return + # Check whether or not fetch-metadata has been run at all metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") if not os.path.exists(repomd_fname): msg = f"{repomd_fname!r} does not exist. Run `fedmod fetch-metadata`." raise MissingMetadata(msg) - repomd_xml = etree.parse(repomd_fname) - repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") - if repo_relative_modulemd is None: - msg = (f"No 'modules' entry found in {repomd_fname!r}. " - "Is the metadata for a non-modular repo?") - raise MissingMetadata(msg) - repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) - if not os.path.exists(repo_modulemd_fname): - msg = (f"{repo_modulemd_fname!r} does not exist. " - "Try running `fedmod fetch-metadata` again.") - raise MissingMetadata(msg) - with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: - modules_yaml = modules_yaml_gz.read() - modules = modulemd.loads_all(modules_yaml) - for module in modules: - _MODULE_FORWARD_LOOKUP[module.name] = module - for srpmname in module.components.rpms: - srpm_entry = _SRPM_REVERSE_LOOKUP.setdefault(srpmname, []) - srpm_entry.append(module.name) - for rpmname in module.artifacts.rpms: - rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] - rpm_entry = _RPM_REVERSE_LOOKUP.setdefault(rpmprefix, []) - rpm_entry.append(module.name) - # Read the extra RPM bootstrap metadata - if not os.path.exists(_BOOTSTRAP_REVERSE_LOOKUP_CACHE): - msg = (f"{_BOOTSTRAP_REVERSE_LOOKUP_CACHE!r} does not exist. " - "Try running `fedmod fetch-metadata` again.") - raise MissingMetadata(msg) - with open(_BOOTSTRAP_REVERSE_LOOKUP_CACHE, "r") as cachefile: - _BOOTSTRAP_COMPONENTS.update(json.load(cachefile)) + # Check whether or not fetch-metadata actually finished + for cache_entry in _LOOKUP_CACHES.values(): + if not os.path.exists(cache_entry): + msg = (f"{cache_entry!r} does not exist. " + "Try running `fedmod fetch-metadata` again.") + raise MissingMetadata(msg) + # Load the metadata + # TODO: Switch to lazy loading of the actual data + _SRPM_REVERSE_LOOKUP.update(_read_cache("_SRPM_REVERSE_LOOKUP_CACHE")) + _RPM_REVERSE_LOOKUP.update(_read_cache("_RPM_REVERSE_LOOKUP_CACHE")) + _BOOTSTRAP_COMPONENTS.update(_read_cache("_BOOTSTRAP_COMPONENTS_CACHE")) + _MODULE_FORWARD_LOOKUP.update(_read_cache("_MODULE_FORWARD_LOOKUP_CACHE")) def list_modules(): return _MODULE_FORWARD_LOOKUP.keys() def get_rpms_in_module(module_name): - if module_name in _MODULE_FORWARD_LOOKUP: - return _MODULE_FORWARD_LOOKUP[module_name].artifacts.rpms - return set() + return _MODULE_FORWARD_LOOKUP.get(module_name, []) def get_modules_for_rpm(rpm_name): result = _RPM_REVERSE_LOOKUP.get(rpm_name)