From 9368adbf944e9ee863d99b505f12558e21bda03f Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 1/16] When waiting for repositories, explictly request them With recent versions of Koji, repositories are no longer automatically regenerated when stale, but instead a user has to explicitly submit a request. So, do this: - When building RPMs in Koji, before the next batch - When waiting for the app package repo when building a container locally or on the server. See: https://pagure.io/koji/blob/master/f/docs/source/repo_generation.rst --- diff --git a/flatpak_module_tools/build_scheduler.py b/flatpak_module_tools/build_scheduler.py index 819ee78..f5554fc 100644 --- a/flatpak_module_tools/build_scheduler.py +++ b/flatpak_module_tools/build_scheduler.py @@ -71,6 +71,7 @@ class RepoWaiter: async def wait_for_event(self, event): if self.last_repo_event < event: + self.session.repo.request(self.tag, min_event=event) while True: next_repo_event = await self.get_next_repo_event() if next_repo_event >= event: diff --git a/flatpak_module_tools/rpm_builder.py b/flatpak_module_tools/rpm_builder.py index c391931..c26d83a 100644 --- a/flatpak_module_tools/rpm_builder.py +++ b/flatpak_module_tools/rpm_builder.py @@ -8,6 +8,7 @@ from typing import Any, Collection, Dict, List, Tuple import click import koji +from koji_cli.lib import activate_session import networkx from .build_scheduler import KojiBuildScheduler, MockBuildScheduler @@ -380,10 +381,18 @@ class RpmBuilder: if wait_for_event >= 0: with Status("Waiting for repository with necessary packages"): session = self.profile.koji_session + first = True while True: repo_info = session.getRepo(package_tag, dist=package_dist_repo) if repo_info["create_event"] >= wait_for_event: break + if first and not package_dist_repo: + # dist repos are automatically created through the + # tag2distrepo plugin, so we don't need to request them + # (and can't - it's not currently supported by repo.request) + activate_session(session, self.profile.koji_options) + self.session.repo.request(package_tag, min_event=wait_for_event) + first = False time.sleep(20) def _prompt_for_rebuild(self, manual_packages: Collection[str], From 759b404099ab2f608e87f246a1618d64f03dc480 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 2/16] Allow local RPMs in local runtime builds For development purposes, it is helpful to be able to use the local RPM repo ($arch/rpms) for testing changes to runtimes. While FMT cannot currently build these, if they are built and placed in that directory with repodata, they should be usable. For them to be found, the priorities of the runtime-packages and local repos must match, as they do for the app and local repos in app builds. (When repos have priorities, the RPM from the lowest priority repo will take precedence over that of a higher priority, even if the latter has a higher ENVR.) --- diff --git a/flatpak_module_tools/build_context.py b/flatpak_module_tools/build_context.py index af428d4..e18acba 100644 --- a/flatpak_module_tools/build_context.py +++ b/flatpak_module_tools/build_context.py @@ -152,7 +152,7 @@ class BuildContext(ABC): if for_container: if self.flatpak_spec.build_runtime: - repos.append(self.runtime_package_repo.dnf_config(priority=10)) + repos.append(self.runtime_package_repo.dnf_config(priority=20)) else: repos.append(self.runtime_package_repo.dnf_config( priority=10, includepkgs=self.runtime_packages diff --git a/tests/test_build_context.py b/tests/test_build_context.py index 155e3c0..fdcc13f 100644 --- a/tests/test_build_context.py +++ b/tests/test_build_context.py @@ -210,7 +210,7 @@ def test_auto_build_context_runtime(runtime_container_spec, profile: ProfileConf baseurl=https://kojifiles.example.com/repos/f39-flatpak-runtime-packages/latest/$basearch/ enabled=1 skip_if_unavailable=False - priority=10 + priority=20 """) ] From 6318d74c2f01460a3df20f673844266f1d03d1b6 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 3/16] Support building extensions Since extensions will generally require reorganizing files to fit the necessary layout, and because (at least application) extensions may be mounted to an arbitrary location, only runtime packages are used in building extensions. Extension container.yaml need to include at least `build-extension`, `runtime-version`, and `branch`, should list `packages` as normal, and then include a `cleanup-commands` to move the correct files to the necessary layout under /app. --- diff --git a/flatpak_module_tools/build_context.py b/flatpak_module_tools/build_context.py index e18acba..68b7a65 100644 --- a/flatpak_module_tools/build_context.py +++ b/flatpak_module_tools/build_context.py @@ -128,7 +128,7 @@ class BuildContext(ABC): @property def release(self): """Bare number for the operating system release (e.g. 39 for Fedora 39)""" - if self.flatpak_spec.build_runtime: + if self.flatpak_spec.build_runtime or self.flatpak_spec.build_extension: runtime_version = self.flatpak_spec.branch else: runtime_version = self.runtime_info.version @@ -151,7 +151,7 @@ class BuildContext(ABC): local_repo_path = self.local_repo if for_container: - if self.flatpak_spec.build_runtime: + if self.flatpak_spec.build_runtime or self.flatpak_spec.build_extension: repos.append(self.runtime_package_repo.dnf_config(priority=20)) else: repos.append(self.runtime_package_repo.dnf_config( @@ -159,7 +159,7 @@ class BuildContext(ABC): )) repos.append(self.app_package_repo.dnf_config(priority=20)) else: - if self.flatpak_spec.build_runtime: + if self.flatpak_spec.build_runtime or self.flatpak_spec.build_extension: raise NotImplementedError("Runtime package building is not implemented") repos.append(self.app_build_repo.dnf_config(priority=20)) @@ -198,7 +198,7 @@ class AutoBuildContext(BuildContext): @cached_property def nvr(self): - if self.flatpak_spec.build_runtime: + if self.flatpak_spec.build_runtime or self.flatpak_spec.build_extension: name = self.container_spec.flatpak.component or self.container_spec.flatpak.name return f"{name}-{self.container_spec.flatpak.branch}-1" else: diff --git a/flatpak_module_tools/cli.py b/flatpak_module_tools/cli.py index a714c6c..ab8ca60 100644 --- a/flatpak_module_tools/cli.py +++ b/flatpak_module_tools/cli.py @@ -218,7 +218,7 @@ def build_container(ctx, target=target ) - if not container_spec.flatpak.build_runtime: + if not container_spec.flatpak.build_runtime and not container_spec.flatpak.build_extension: rpm_builder = RpmBuilder(build_context, workdir=paths.workdir) rpm_builder.check(include_localrepo=False, allow_outdated=allow_outdated) @@ -289,7 +289,7 @@ def build_container_local(ctx, local_runtime=local_runtime, target=target ) - if not container_spec.flatpak.build_runtime: + if not container_spec.flatpak.build_runtime and not container_spec.flatpak.build_extension: rpm_builder = RpmBuilder(build_context, workdir=paths.workdir) rpm_builder.check(include_localrepo=True, allow_outdated=allow_outdated) diff --git a/flatpak_module_tools/container_builder.py b/flatpak_module_tools/container_builder.py index e5f31ff..f31365f 100644 --- a/flatpak_module_tools/container_builder.py +++ b/flatpak_module_tools/container_builder.py @@ -277,7 +277,7 @@ class ContainerBuilder: info(f" wrote {outname_base}.config.json") def _create_rpm_manifest(self, outname_base: Path): - if self.context.flatpak_spec.build_runtime: + if self.context.flatpak_spec.build_runtime or self.context.flatpak_spec.build_extension: restrict_to = None else: restrict_to = self.executor.absolute_installroot / "app" @@ -294,7 +294,7 @@ class ContainerBuilder: self.executor = executor - if self.context.flatpak_spec.build_runtime: + if self.context.flatpak_spec.build_runtime or self.context.flatpak_spec.build_extension: runtime_info = None else: runtime_info = self.context.runtime_info @@ -369,7 +369,7 @@ class ContainerBuilder: def assemble(self, *, installroot: Path, workdir: Path, resultdir: Path): - if self.context.flatpak_spec.build_runtime: + if self.context.flatpak_spec.build_runtime or self.context.flatpak_spec.build_extension: runtimever = self.context.nvr.rsplit('-', 2)[1] else: runtimever = self.context.runtime_info.version @@ -400,7 +400,7 @@ class ContainerBuilder: installroot = Path("/contents") - if self.context.flatpak_spec.build_runtime: + if self.context.flatpak_spec.build_runtime or self.context.flatpak_spec.build_extension: runtimever = self.context.nvr.rsplit('-', 2)[1] else: runtimever = self.context.runtime_info.version diff --git a/flatpak_module_tools/container_spec.py b/flatpak_module_tools/container_spec.py index a818dd8..61733e7 100644 --- a/flatpak_module_tools/container_spec.py +++ b/flatpak_module_tools/container_spec.py @@ -159,6 +159,7 @@ class FlatpakSpec(BaseSpec): self.base_image = self._get_str('base_image', None) self.branch = self._get_str('branch', 'stable') self.build_runtime = self._get_bool('build-runtime', False) + self.build_extension = self._get_bool('build-extension', False) self.cleanup_commands = self._get_str('cleanup-commands', None) self.command = self._get_str('command', None) self.component = self._get_str('component', None) @@ -250,7 +251,7 @@ class ContainerSpec(BaseSpec): ) ) else: - if self.flatpak.build_runtime: + if self.flatpak.build_runtime or self.flatpak.build_extension: required_attrs = NEW_STYLE_ATTRS_RUNTME else: required_attrs = NEW_STYLE_ATTRS diff --git a/flatpak_module_tools/flatpak_builder.py b/flatpak_module_tools/flatpak_builder.py index 373e47f..e7509e4 100644 --- a/flatpak_module_tools/flatpak_builder.py +++ b/flatpak_module_tools/flatpak_builder.py @@ -293,6 +293,7 @@ class FileTreeProcessor: class BaseFlatpakSourceInfo(ABC): runtime: bool + extension: bool spec: FlatpakSpec @abstractmethod @@ -335,6 +336,8 @@ class ModuleFlatpakSourceInfo(BaseFlatpakSourceInfo): # A runtime module must have a 'runtime' profile, but can have other # profiles for SDKs, minimal runtimes, etc. self.runtime = 'runtime' in base_module.mmd.get_profile_names() + # Extension support not implemented for module backend + self.extension = False if profile is None: profile = 'runtime' if self.runtime else 'default' @@ -502,11 +505,12 @@ class PackageFlatpakSourceInfo(BaseFlatpakSourceInfo): def __init__(self, spec: FlatpakSpec, runtime_info: Optional[RuntimeInfo]): if spec.build_runtime and runtime_info: raise RuntimeError("runtime_info can only be set for an application") - if not spec.build_runtime and not runtime_info: + if not spec.build_runtime and not spec.build_extension and not runtime_info: raise RuntimeError("runtime_info must be set for an application") self.spec = spec self.runtime = spec.build_runtime self.runtime_info = runtime_info + self.extension = spec.build_extension def precheck(self): pass @@ -753,7 +757,7 @@ class FlatpakBuilder: def get_components(self, manifest): all_components = self._get_components(manifest) - if self.source.runtime: + if self.source.runtime or self.source.extension: image_components = all_components else: image_components = self.source.filter_app_manifest(all_components) @@ -767,7 +771,7 @@ class FlatpakBuilder: if spec.finish_args: # shlex.split(None) reads from standard input, so avoid that finish_args = shlex.split(spec.finish_args, comments=True) - if spec.command and not self.source.runtime: + if spec.command and not self.source.runtime and not self.source.extension: finish_args = ['--command', spec.command] + finish_args subprocess.check_call(['flatpak', 'build-finish'] + finish_args + [builddir]) @@ -805,14 +809,15 @@ class FlatpakBuilder: 'runtime_id': runtime_id, 'sdk_id': sdk_id, 'arch': self.arch.flatpak, + 'runtime_version': spec.runtime_version or branch, 'branch': branch } METADATA_TEMPLATE = dedent("""\ [Runtime] name={id} - runtime={runtime_id}/{arch}/{branch} - sdk={sdk_id}/{arch}/{branch} + runtime={runtime_id}/{arch}/{runtime_version} + sdk={sdk_id}/{arch}/{runtime_version} """) with open(os.path.join(builddir, 'metadata'), 'w') as f: @@ -1052,7 +1057,7 @@ class FlatpakBuilder: def build_container(self, tarred_filesystem: str, tar_outfile: bool = True): outfile = os.path.join(self.workdir, 'flatpak-oci-image') - if self.source.runtime: + if self.source.runtime or self.source.extension: ref_name = self._create_runtime_oci(tarred_filesystem, outfile) else: ref_name = self._create_app_oci(tarred_filesystem, outfile) From 812ad8fb0f48d3c4dcc45828753ada1c76e98635 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 4/16] Support rename-mime-file This allows automatic renaming of a single mime "package" file so that it will be exported once installed. --- diff --git a/flatpak_module_tools/container_spec.py b/flatpak_module_tools/container_spec.py index 61733e7..8b54c0b 100644 --- a/flatpak_module_tools/container_spec.py +++ b/flatpak_module_tools/container_spec.py @@ -173,6 +173,7 @@ class FlatpakSpec(BaseSpec): self.packages = self._get_package_list('packages', []) self.rename_appdata_file = self._get_str('rename-appdata-file', None) self.rename_desktop_file = self._get_str('rename-desktop-file', None) + self.rename_mime_file = self._get_str('rename-mime-file', None) self.rename_icon = self._get_str('rename-icon', None) self.runtime = self._get_str('runtime', None) self.runtime_name = self._get_str('runtime-name', None) diff --git a/flatpak_module_tools/flatpak_builder.py b/flatpak_module_tools/flatpak_builder.py index e7509e4..f84bf03 100644 --- a/flatpak_module_tools/flatpak_builder.py +++ b/flatpak_module_tools/flatpak_builder.py @@ -181,6 +181,18 @@ class FileTreeProcessor: tree.write(self.appdata_file, encoding="UTF-8", xml_declaration=True) + def _rename_mime_file(self): + if not self.spec.rename_mime_file: + return + + mimepkgs_dir = os.path.join(self.app_root, "share", "mime", "packages") + src = os.path.join(mimepkgs_dir, self.spec.rename_mime_file) + mime_basename = self.spec.app_id + ".xml" + dest = os.path.join(mimepkgs_dir, mime_basename) + + self.log.info("Renaming %s to %s", self.spec.rename_mime_file, mime_basename) + os.rename(src, dest) + def _rename_icon(self): if not self.spec.rename_icon: return @@ -286,6 +298,7 @@ class FileTreeProcessor: def process(self): self._process_appdata_file() self._rename_desktop_file() + self._rename_mime_file() self._rename_icon() self._rewrite_desktop_file() self._compose_appstream() diff --git a/flatpak_module_tools/flatpak_generator.py b/flatpak_module_tools/flatpak_generator.py index e90b712..14ccf4f 100644 --- a/flatpak_module_tools/flatpak_generator.py +++ b/flatpak_module_tools/flatpak_generator.py @@ -114,6 +114,7 @@ class FlatpakGenerator(str): 'desktop-file-name-suffix', 'rename-appdata-file', 'rename-desktop-file', + 'rename-mime-file', 'rename-icon', 'copy-icon']: if key in manifest: From 543105f562b660c94acf6ed2adccd44373d4a170 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 5/16] Support add-extensions This handles both the creation of mount point directories (instead of requiring "mkdir -p" in cleanup-commands) as well as the build-finish --extension arguments (instead of adding them to finish-args). --- diff --git a/flatpak_module_tools/container_builder.py b/flatpak_module_tools/container_builder.py index f31365f..6baa83f 100644 --- a/flatpak_module_tools/container_builder.py +++ b/flatpak_module_tools/container_builder.py @@ -245,6 +245,14 @@ class ContainerBuilder: mounts=mounts, enable_network=True) def _cleanup_tree(self, builder: FlatpakBuilder): + extdirs = [] + for ext in self.context.flatpak_spec.add_extensions: + extdirs += [os.path.join(self.executor.installroot, + "usr" if self.context.flatpak_spec.build_runtime else "app", + ext.directory)] + if extdirs: + self.executor.check_call(["mkdir", "-p"] + extdirs) + script = builder.get_cleanup_script() if not script or script.strip() == "": return diff --git a/flatpak_module_tools/container_spec.py b/flatpak_module_tools/container_spec.py index 8b54c0b..cc615da 100644 --- a/flatpak_module_tools/container_spec.py +++ b/flatpak_module_tools/container_spec.py @@ -136,7 +136,37 @@ class PackageSpec(BaseSpec): self.platforms = None +class ExtensionSpec(BaseSpec): + def __init__(self, path, extname, yaml_object): + super().__init__(path + "/" + extname, yaml_object) + # string, mandatory + self.directory = self._get_str('directory') + self.args = [f"--extension={extname}=directory={self.directory}"] + # string, optional + for arg in ('version', 'versions', 'add-ld-path', 'merge-dirs', 'download-if', + 'autoprune-unless', 'enable-if', 'subdirectory-suffix'): + val = self._get_str(arg, None) + if val: + self.args += [f"--extension={extname}={arg}={val}"] + # boolean, optional + for arg in ('subdirectories', 'no-autodownload', 'autodelete'): + val = self._get_bool(arg, False) + if val: + self.args += [f"--extension={extname}={arg}=true"] + + class FlatpakSpec(BaseSpec): + def _get_extension_items(self, key, default) -> List["ExtensionSpec"]: + def type_convert(val): + if isinstance(val, dict) and all(isinstance(v, dict) for i, v in val.items()): + return [ + ExtensionSpec(f"{self.path}", i, v) for i, v in val.items() + ] + else: + raise ValidationError(f"{self.path}: {key} must be a mapping of mappings") + + return self._get(key, type_convert, default) + def _get_package_list(self, key, default) -> List["PackageSpec"]: def type_convert(val): if isinstance(val, List) and all(isinstance(v, (str, dict)) for v in val): @@ -151,6 +181,7 @@ class FlatpakSpec(BaseSpec): def __init__(self, path, flatpak_yaml): super().__init__(path, flatpak_yaml) + self.add_extensions = self._get_extension_items('add-extensions', []) self.app_id = self._get_str("id") self.appdata_license = self._get_str('appdata-license', None) self.appstream_compose = self._get_bool('appstream-compose', True) diff --git a/flatpak_module_tools/flatpak_builder.py b/flatpak_module_tools/flatpak_builder.py index f84bf03..87ae5ee 100644 --- a/flatpak_module_tools/flatpak_builder.py +++ b/flatpak_module_tools/flatpak_builder.py @@ -784,9 +784,12 @@ class FlatpakBuilder: if spec.finish_args: # shlex.split(None) reads from standard input, so avoid that finish_args = shlex.split(spec.finish_args, comments=True) + for ext in spec.add_extensions: + finish_args += ext.args if spec.command and not self.source.runtime and not self.source.extension: finish_args = ['--command', spec.command] + finish_args + self.log.debug("Calling flatpak build-finish %s %s", " ".join(finish_args), builddir) subprocess.check_call(['flatpak', 'build-finish'] + finish_args + [builddir]) def _create_repo(self): diff --git a/tests/test_container_spec.py b/tests/test_container_spec.py index 384bccc..6ec3f02 100644 --- a/tests/test_container_spec.py +++ b/tests/test_container_spec.py @@ -101,6 +101,15 @@ flatpak: component: flatpak-runtime branch: f39 sdk: org.fedoraproject.Sdk + add-extensions: + org.fedoraproject.Platform.hunspell: + directory: share/myspell/dicts + versions: stable;f39 + org.fedoraproject.Platform.openh264: + directory: lib64/openh264 + version: '2.4.1' + add-ld-path: extra + autodelete: true finish-args: >- --env=GI_TYPELIB_PATH=/app/lib64/girepository-1.0 --env=GST_PLUGIN_SYSTEM_PATH=/app/lib64/gstreamer-1.0:/usr/lib64/gstreamer-1.0 @@ -182,6 +191,18 @@ def test_runtime_container_spec(tmp_path): "--env=GI_TYPELIB_PATH=/app/lib64/girepository-1.0 " "--env=GST_PLUGIN_SYSTEM_PATH=/app/lib64/gstreamer-1.0:/usr/lib64/gstreamer-1.0" ) + assert spec.flatpak.add_extensions[0].args == [ + "--extension=org.fedoraproject.Platform.hunspell=directory=share/myspell/dicts", + "--extension=org.fedoraproject.Platform.hunspell=versions=stable;f39", + ] + assert spec.flatpak.add_extensions[0].directory == "share/myspell/dicts" + assert spec.flatpak.add_extensions[1].args == [ + "--extension=org.fedoraproject.Platform.openh264=directory=lib64/openh264", + "--extension=org.fedoraproject.Platform.openh264=version=2.4.1", + "--extension=org.fedoraproject.Platform.openh264=add-ld-path=extra", + "--extension=org.fedoraproject.Platform.openh264=autodelete=true", + ] + assert spec.flatpak.add_extensions[1].directory == "lib64/openh264" assert spec.flatpak.cleanup_commands == dedent("""\ mv -f /usr/bin/flatpak-xdg-email /usr/bin/xdg-email mv -f /usr/bin/flatpak-xdg-open /usr/bin/xdg-open From b556623d32aeed3534d4f55e3f6fc78bdace89a1 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 6/16] Support rename-mime-icons This handles both the renaming of a list of mimetype icons as well as the corresponding modification to the mime package. --- diff --git a/flatpak_module_tools/container_spec.py b/flatpak_module_tools/container_spec.py index cc615da..5231661 100644 --- a/flatpak_module_tools/container_spec.py +++ b/flatpak_module_tools/container_spec.py @@ -206,6 +206,7 @@ class FlatpakSpec(BaseSpec): self.rename_desktop_file = self._get_str('rename-desktop-file', None) self.rename_mime_file = self._get_str('rename-mime-file', None) self.rename_icon = self._get_str('rename-icon', None) + self.rename_mime_icons = self._get_str_list('rename-mime-icons', []) self.runtime = self._get_str('runtime', None) self.runtime_name = self._get_str('runtime-name', None) self.runtime_version = self._get_str('runtime-version', None) diff --git a/flatpak_module_tools/flatpak_builder.py b/flatpak_module_tools/flatpak_builder.py index 87ae5ee..c802a5c 100644 --- a/flatpak_module_tools/flatpak_builder.py +++ b/flatpak_module_tools/flatpak_builder.py @@ -193,10 +193,7 @@ class FileTreeProcessor: self.log.info("Renaming %s to %s", self.spec.rename_mime_file, mime_basename) os.rename(src, dest) - def _rename_icon(self): - if not self.spec.rename_icon: - return - + def _rename_icon_helper(self, icon, mime_icon=False): found_icon = False icons_dir = os.path.join(self.app_root, "share/icons") @@ -205,15 +202,18 @@ class FileTreeProcessor: depth = relative.count("/") for source_file in filenames: - if source_file.startswith(self.spec.rename_icon): + if source_file.startswith(icon): source_path = os.path.join(full_dir, source_file) is_file = os.path.isfile(source_path) - extension = source_file[len(self.spec.rename_icon):] + extension = source_file[len(icon):] if is_file and depth == 3 and (extension.startswith(".") or extension.startswith("-symbolic")): found_icon = True - new_name = self.spec.app_id + extension + if mime_icon: + new_name = self.spec.app_id + "." + source_file + else: + new_name = self.spec.app_id + extension self.log.info("%s icon %s/%s to %s/%s", "Copying" if self.spec.copy_icon else "Renaming", @@ -227,18 +227,27 @@ class FileTreeProcessor: os.rename(source_path, dest_path) else: if not is_file: - self.log.debug("%s/%s matches 'rename-icon', but not a regular file", - full_dir, source_file) + self.log.debug("%s/%s matches '%s', but not a regular file", + full_dir, source_file, icon) elif depth != 3: - self.log.debug("%s/%s matches 'rename-icon', but not at depth 3", - full_dir, source_file) + self.log.debug("%s/%s matches '%s', but not at depth 3", + full_dir, source_file, icon) else: - self.log.debug("%s/%s matches 'rename-icon', but name does not " + self.log.debug("%s/%s matches '%s', but name does not " "continue with '.' or '-symbolic.'", - full_dir, source_file) + full_dir, source_file, icon) if not found_icon: - raise RuntimeError(f"icon {self.spec.rename_icon} not found below {icons_dir}") + raise RuntimeError(f"icon {icon} not found below {icons_dir}") + + + def _rename_icon(self): + if self.spec.rename_icon: + self._rename_icon_helper(self.spec.rename_icon) + + def _rename_mime_icons(self): + for icon in self.spec.rename_mime_icons: + self._rename_icon_helper(icon, mime_icon=True) def _rewrite_desktop_file(self): if not ( @@ -284,6 +293,42 @@ class FileTreeProcessor: with open(desktop, "w") as f: cp.write(f, space_around_delimiters=False) + def _rewrite_mime_file(self): + if not self.spec.rename_mime_icons: + return + + xmlns = 'http://www.freedesktop.org/standards/shared-mime-info' + xmlns_ = '{' + xmlns + '}' + + mime_file = os.path.join(self.app_root, "share/mime/packages", self.spec.app_id + ".xml") + self.log.debug("Rewriting contents of %s", mime_file) + try: + tree = ElementTree.parse(mime_file) + root = tree.getroot() + mime_modified = False + for mime_type in root.findall(xmlns_ + 'mime-type'): + icon = mime_type.find(xmlns_ + 'icon') + if icon is not None: + icon_name = icon.get('name') + icon.set('name', self.spec.app_id + "." + icon_name) + mime_modified = True + else: + icon = mime_type.find(xmlns_ + 'generic-icon') + if icon is not None: + icon_name = icon.get('name') + icon.set('name', self.spec.app_id + "." + icon_name) + mime_modified = True + else: + icon = ElementTree.SubElement(mime_type, 'generic-icon') + icon_name = mime_type.get('type').replace('/', '-') + icon.set('name', self.spec.app_id + "." + icon_name) + mime_modified = True + if mime_modified: + ElementTree.register_namespace("", xmlns) + tree.write(mime_file, encoding="UTF-8", xml_declaration=True) + except: + pass + def _compose_appstream(self): if not self.spec.appstream_compose or not self.appdata_file: return @@ -300,7 +345,9 @@ class FileTreeProcessor: self._rename_desktop_file() self._rename_mime_file() self._rename_icon() + self._rename_mime_icons() self._rewrite_desktop_file() + self._rewrite_mime_file() self._compose_appstream() diff --git a/flatpak_module_tools/flatpak_generator.py b/flatpak_module_tools/flatpak_generator.py index 14ccf4f..ec858b0 100644 --- a/flatpak_module_tools/flatpak_generator.py +++ b/flatpak_module_tools/flatpak_generator.py @@ -116,6 +116,7 @@ class FlatpakGenerator(str): 'rename-desktop-file', 'rename-mime-file', 'rename-icon', + 'rename-mime-icons', 'copy-icon']: if key in manifest: yml['flatpak'][key] = manifest[key] From 30a9d20ec24015afa54b872d494a33621cdfc2a2 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 7/16] Update generator for Flathub v2 API This fixes flathub-module init --flathub after the decommissioning of the Flathub v1 API. The apps endpoint no longer exists to allow our own search, so we have to use the new API to run the search server-side. However, because using a complete ID will broaden rather than narrow the search results due to hits from the TLD (e.g. com., io., org.), if the search term resembles an ID, we first check if that is valid first. Therefore, the workflow will often necessitate two attempts, the first with a name search to find possible ID matches, and the second with the full ID. --- diff --git a/flatpak_module_tools/flatpak_generator.py b/flatpak_module_tools/flatpak_generator.py index ec858b0..a18e88a 100644 --- a/flatpak_module_tools/flatpak_generator.py +++ b/flatpak_module_tools/flatpak_generator.py @@ -42,16 +42,21 @@ yaml.add_representer(NoSortMapping, _represent_no_sort_mapping) def _load_flathub_manifest(search_term): - response = requests.get("https://flathub.org/api/v1/apps") - response.raise_for_status() - apps = response.json() - matches = [] - search_lower = search_term.lower() - for app in apps: - if (search_lower in app['flatpakAppId'].lower() or - search_lower in app['name'].lower()): - matches.append((app['flatpakAppId'], app['name'])) + # first, see if the complete id was passed + if search_term.count('.') >= 2: + response = requests.get(f"https://flathub.org/api/v2/summary/{search_term}") + if response.status_code == 200: + matches.append((search_term, search_term)) + + if len(matches) == 0: + search_data = {'query': search_term.lower()} + response = requests.post("https://flathub.org/api/v2/search", data=json.dumps(search_data)) + response.raise_for_status() + apps = response.json() + + for app in apps['hits']: + matches.append((app['app_id'], app['name'])) if len(matches) > 1: max_id_len = max([len(app_id) for app_id, _ in matches]) diff --git a/tests/files/generator/apps-multiple.json b/tests/files/generator/apps-multiple.json new file mode 100644 index 0000000..d7958b2 --- /dev/null +++ b/tests/files/generator/apps-multiple.json @@ -0,0 +1,32 @@ +{ + "estimatedTotalHits": 2, + "hits": [ + { + "app_id": "org.gnome.Crosswords", + "arches": [ + "aarch64", + "x86_64" + ], + "id": "org_gnome_Crosswords", + "name": "Crosswords", + "project_license": "GPL-3.0-or-later", + "runtime": "org.gnome.Platform/x86_64/46", + "summary": "Solve crossword puzzles", + "type": "desktop-application" + }, + { + "app_id": "org.gnome.Crosswords.Editor", + "arches": [ + "aarch64", + "x86_64" + ], + "id": "org_gnome_Crosswords_Editor", + "name": "Crossword Editor", + "project_license": "GPL-3.0-or-later", + "runtime": "org.gnome.Platform/x86_64/46", + "summary": "Create crossword puzzles", + "type": "desktop-application" + } + ], + "query": "crossword" +} diff --git a/tests/files/generator/apps-noent.json b/tests/files/generator/apps-noent.json new file mode 100644 index 0000000..144b8fc --- /dev/null +++ b/tests/files/generator/apps-noent.json @@ -0,0 +1,6 @@ +{ + "estimatedTotalHits": 0, + "hits": [ + ], + "query": "foobar" +} diff --git a/tests/files/generator/apps.json b/tests/files/generator/apps.json index bf8df16..22bfd75 100644 --- a/tests/files/generator/apps.json +++ b/tests/files/generator/apps.json @@ -1,7 +1,19 @@ -[{"flatpakAppId":"org.gnome.eog", - "name":"Eye of GNOME", - "summary":"Browse and rotate images", - "iconDesktopUrl":"/repo/appstream/x86_64/icons/128x128/org.gnome.eog.png"}, -{"flatpakAppId":"org.gnome.FeedReader", - "name":"FeedReader","summary":"RSS client for various webservices", - "iconDesktopUrl":"/repo/appstream/x86_64/icons/128x128/org.gnome.FeedReader.png"}] +{ + "estimatedTotalHits": 1, + "hits": [ + { + "app_id": "org.gnome.eog", + "arches": [ + "aarch64", + "x86_64" + ], + "id": "org_gnome_eog", + "name": "Eye of GNOME", + "project_license": "GPL-2.0+ and GFDL-1.3", + "runtime": "org.gnome.Platform/x86_64/46", + "summary": "Browse and rotate images", + "type": "desktop-application" + } + ], + "query": "eog" +} diff --git a/tests/test_flatpak_generator.py b/tests/test_flatpak_generator.py index 79817dd..3ad0861 100644 --- a/tests/test_flatpak_generator.py +++ b/tests/test_flatpak_generator.py @@ -20,6 +20,12 @@ testfiles_dir = os.path.join(os.path.dirname(__file__), 'files', 'generator') with open(os.path.join(testfiles_dir, 'apps.json')) as f: APPS_JSON = f.read() +with open(os.path.join(testfiles_dir, 'apps-multiple.json')) as f: + APPS_MULTIPLE_JSON = f.read() + +with open(os.path.join(testfiles_dir, 'apps-noent.json')) as f: + APPS_NOENT_JSON = f.read() + with open(os.path.join(testfiles_dir, 'eog.yaml')) as f: EOG_YAML = f.read() @@ -85,21 +91,23 @@ class TestFlatpak(object): @responses.activate @pytest.mark.needs_metadata - @pytest.mark.parametrize(('search_term', 'extension', 'expected_error'), + @pytest.mark.parametrize(('search_term', 'response_body', 'extension', 'expected_error'), [ - ('org.gnome.eog', 'yaml', None), - ('org.gnome.eog', 'yml', None), - ('org.gnome.eog', 'json', None), - ('eYe of gNome', 'yaml', None), - ('org.gnome', 'yaml', + ('org.gnome.eog', APPS_JSON, 'yaml', None), + ('org.gnome.eog', APPS_JSON, 'yml', None), + ('org.gnome.eog', APPS_JSON, 'json', None), + ('eYe of gNome', APPS_JSON, 'yaml', None), + ('org.gnome', APPS_MULTIPLE_JSON, 'yaml', 'Multiple matches found on flathub.org'), - ('notexist', 'yaml', + ('notexist', APPS_NOENT_JSON, 'yaml', 'No match found on flathub.org'), ]) - def test_flatpak_from_flathub(self, search_term, extension, + def test_flatpak_from_flathub(self, search_term, response_body, extension, expected_error): - responses.add(responses.GET, 'https://flathub.org/api/v1/apps', - body=APPS_JSON, content_type='application/json') + responses.add(responses.POST, 'https://flathub.org/api/v2/search', + body=response_body, content_type='application/json') + responses.add(responses.GET, 'https://flathub.org/api/v2/summary/org.gnome.eog', + status=200) responses.add(responses.GET, 'https://bodhi.fedoraproject.org/releases/?state=current', body=RELEASES_JSON, content_type='application/json') From 695befcbe48ca8367c9b360b7469f6cbd57ef31c Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 8/16] Support extra-data This allows creating flatpaks which download non-redistributable content during install. For Fedora, the primary use case is for an openh264 extension which fetches the binaries from Cisco. The syntax is a list of mappings, similar in syntax to extra-data sources in flatpak-builder yaml: flatpak: ... extra-data: - filename: ... url: ... sha256: ... size: ... only-arches: [ARCH] ... Based partially on work by Kalev Lember in: https://pagure.io/flatpak-module-tools/pull-request/41 --- diff --git a/flatpak_module_tools/container_spec.py b/flatpak_module_tools/container_spec.py index 5231661..8508f5f 100644 --- a/flatpak_module_tools/container_spec.py +++ b/flatpak_module_tools/container_spec.py @@ -79,6 +79,36 @@ class BaseSpec: return self._get(key, type_convert, default) @overload + def _get_uint(self, key: str, default: Literal[Option.REQUIRED]) -> int: + ... + + @overload + def _get_uint(self, key: str) -> int: + ... + + @overload + def _get_uint(self, key: str, default: int) -> int: + ... + + @overload + def _get_uint(self, key: str, default: None) -> Optional[int]: + ... + + def _get_uint( + self, key: str, default: Union[Literal[Option.REQUIRED], int, None] = Option.REQUIRED + ) -> Optional[int]: + def type_convert(val): + if isinstance(val, (str, int, float)): + ret = int(val) + if ret < 0: + raise ValidationError(f"{self.path}: {key} must be a non-negative integer") + return ret + else: + raise ValidationError(f"{self.path}: {key} must be a non-negative integer") + + return self._get(key, type_convert, default) + + @overload def _get_str_list(self, key: str, default: Literal[Option.REQUIRED], allow_scalar=False) -> List[str]: ... @@ -155,6 +185,24 @@ class ExtensionSpec(BaseSpec): self.args += [f"--extension={extname}={arg}=true"] +class ExtraDataSpec(BaseSpec): + def __init__(self, path, yaml_object): + super().__init__(path, yaml_object) + name = self._get_str('filename') + uri = self._get_str('url') + checksum = self._get_str('sha256') + size = self._get_uint('size') + installed_size = self._get_uint('installed-size', 0) + self.variant_data = [(name.encode("UTF-8") + b'\x00', + int.from_bytes(size.to_bytes(8, byteorder="little", signed=False), byteorder="big"), + int.from_bytes(installed_size.to_bytes(8, byteorder="little", signed=False), byteorder="big"), + bytes.fromhex(checksum), + uri)] + installed_size_arg = str(installed_size) if installed_size > 0 else "" + self.finish_args = [f"--extra-data={name}:{checksum}:{size}:{installed_size_arg}:{uri}"] + self.only_arches = self._get_str_list('only-arches', None) + + class FlatpakSpec(BaseSpec): def _get_extension_items(self, key, default) -> List["ExtensionSpec"]: def type_convert(val): @@ -167,6 +215,17 @@ class FlatpakSpec(BaseSpec): return self._get(key, type_convert, default) + def _get_extra_data_list(self, key, default) -> List["ExtraDataSpec"]: + def type_convert(val): + if isinstance(val, List) and all(isinstance(v, dict) for v in val): + return [ + ExtraDataSpec(f"{self.path}/{i}", v) for i, v in enumerate(val) + ] + else: + raise ValidationError(f"{self.path}: {key} must be a list of mappings") + + return self._get(key, type_convert, default) + def _get_package_list(self, key, default) -> List["PackageSpec"]: def type_convert(val): if isinstance(val, List) and all(isinstance(v, (str, dict)) for v in val): @@ -199,6 +258,7 @@ class FlatpakSpec(BaseSpec): self.desktop_file_name_suffix = self._get_str('desktop-file-name-suffix', None) self.end_of_life = self._get_str('end-of-life', None) self.end_of_life_rebase = self._get_str('end-of-life-rebase', None) + self.extra_data = self._get_extra_data_list('extra-data', []) self.finish_args = self._get_str('finish-args', None) self.name = self._get_str('name', None) self.packages = self._get_package_list('packages', []) diff --git a/flatpak_module_tools/flatpak_builder.py b/flatpak_module_tools/flatpak_builder.py index c802a5c..da64b49 100644 --- a/flatpak_module_tools/flatpak_builder.py +++ b/flatpak_module_tools/flatpak_builder.py @@ -28,6 +28,8 @@ import tarfile from textwrap import dedent from xml.etree import ElementTree +from gi.repository import GLib + from .container_spec import FlatpakSpec from .utils import Arch, RuntimeInfo @@ -833,6 +835,9 @@ class FlatpakBuilder: finish_args = shlex.split(spec.finish_args, comments=True) for ext in spec.add_extensions: finish_args += ext.args + for extdata in spec.extra_data: + if not extdata.only_arches or self.arch.flatpak in extdata.only_arches: + finish_args += extdata.finish_args if spec.command and not self.source.runtime and not self.source.extension: finish_args = ['--command', spec.command] + finish_args @@ -910,6 +915,15 @@ class FlatpakBuilder: commit_args += ['--add-metadata-string', 'ostree.endoflife-rebase=' + spec.end_of_life_rebase] + extra_data_variant_data = [] + for extdata in spec.extra_data: + if not extdata.only_arches or self.arch.flatpak in extdata.only_arches: + extra_data_variant_data += extdata.variant_data + if extra_data_variant_data: + extra_data_variant = GLib.Variant("a(ayttays)", extra_data_variant_data) + commit_args += ['--add-metadata', + 'xa.extra-data-sources=' + extra_data_variant.print_(True)] + subprocess.check_call(['ostree', 'commit'] + commit_args) subprocess.check_call(['ostree', 'summary', '-u', '--repo', repo]) diff --git a/pyproject.toml b/pyproject.toml index bf1dde5..06140b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dynamic = ["version"] dependencies = [ "click", + "PyGObject", "pyyaml", "requests", "rpm", @@ -41,7 +42,6 @@ tests = [ "jinja2", "koji", "networkx", - "PyGObject", "pytest", "pytest-cov", "responses", diff --git a/tests/test_container_spec.py b/tests/test_container_spec.py index 6ec3f02..15356ea 100644 --- a/tests/test_container_spec.py +++ b/tests/test_container_spec.py @@ -93,6 +93,43 @@ flatpak: command: eog """ +EXTENSION_CONTAINER_YAML = """\ +flatpak: + id: org.fedoraproject.Platform.openh264 + build-extension: true + runtime: org.fedoraproject.Platform + runtime-version: f41 + sdk: org.fedoraproject.Sdk + name: openh264 + component: openh264-flatpak + branch: 2.4.1 + packages: + - noopenh264 + extra-data: + - filename: openh264-2.4.1-2.fc41.aarch64.rpm + url: https://codecs.fedoraproject.org/openh264/41/aarch64/os/Packages/o/openh264-2.4.1-2.fc41.aarch64.rpm + sha256: bf8617eaca86dd1cc6fc61cfa3d08da09b0cbd6f6a3318174d58c33b479bd512 + size: 393795 + only-arches: + - aarch64 + - filename: openh264-2.4.1-2.fc41.ppc64le.rpm + url: https://codecs.fedoraproject.org/openh264/41/ppc64le/os/Packages/o/openh264-2.4.1-2.fc41.ppc64le.rpm + sha256: a585ddfaccb3fdc336e532978002273b8c71132092a189f1496a80c841fd92ac + size: 425361 + only-arches: + - ppc64le + - filename: openh264-2.4.1-2.fc41.x86_64.rpm + url: https://codecs.fedoraproject.org/openh264/41/x86_64/os/Packages/o/openh264-2.4.1-2.fc41.x86_64.rpm + sha256: afe17383b2d009aa671332ff9f5ef0a94bd756e086efa6a60c3fe073a7289cd1 + size: 430302 + only-arches: + - x86_64 + cleanup-commands: | + mkdir -p /app/bin + echo -e "#!/bin/sh\\nbsdtar --strip-components=3 -xf *.rpm\\nrm -f *.rpm" > /app/bin/apply_extra + chmod +x /app/bin/apply_extra +""" + RUNTIME_CONTAINER_YAML = """\ flatpak: id: org.fedoraproject.Platform @@ -178,6 +215,29 @@ def test_app_container_spec(tmp_path): assert not spec.platforms.includes_platform("aarch64") +def test_extension_container_spec(tmp_path): + spec = make_spec(tmp_path, EXTENSION_CONTAINER_YAML) + assert spec.flatpak.app_id == "org.fedoraproject.Platform.openh264" + assert spec.flatpak.branch == "2.4.1" + assert spec.flatpak.build_extension is True + assert spec.flatpak.component == "openh264-flatpak" + assert spec.flatpak.name == "openh264" + assert spec.flatpak.runtime == "org.fedoraproject.Platform" + assert spec.flatpak.runtime_version == "f41" + assert spec.flatpak.sdk == "org.fedoraproject.Sdk" + assert spec.flatpak.extra_data[0].finish_args == [ + "--extra-data=openh264-2.4.1-2.fc41.aarch64.rpm:bf8617eaca86dd1cc6fc61cfa3d08da09b0cbd6f6a3318174d58c33b479bd512:393795::https://codecs.fedoraproject.org/openh264/41/aarch64/os/Packages/o/openh264-2.4.1-2.fc41.aarch64.rpm", + ] + assert spec.flatpak.extra_data[0].only_arches == ["aarch64"] + assert spec.flatpak.extra_data[1].finish_args == [ + "--extra-data=openh264-2.4.1-2.fc41.ppc64le.rpm:a585ddfaccb3fdc336e532978002273b8c71132092a189f1496a80c841fd92ac:425361::https://codecs.fedoraproject.org/openh264/41/ppc64le/os/Packages/o/openh264-2.4.1-2.fc41.ppc64le.rpm", + ] + assert spec.flatpak.extra_data[1].only_arches == ["ppc64le"] + assert spec.flatpak.extra_data[2].finish_args == [ + "--extra-data=openh264-2.4.1-2.fc41.x86_64.rpm:afe17383b2d009aa671332ff9f5ef0a94bd756e086efa6a60c3fe073a7289cd1:430302::https://codecs.fedoraproject.org/openh264/41/x86_64/os/Packages/o/openh264-2.4.1-2.fc41.x86_64.rpm", + ] + assert spec.flatpak.extra_data[2].only_arches == ["x86_64"] + def test_runtime_container_spec(tmp_path): spec = make_spec(tmp_path, RUNTIME_CONTAINER_YAML) From 1bcb7df9c8be613ab335918c789eb6f7dd42fd71 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 9/16] RpmBuilder: allow builds to proceed without a mainline Koji build This avoids errors when a package has not yet been imported into Koji, or hasn't reach the configured source_koji_tag. As long as the not-yet-imported (or not-yet-stable) package has been manually built, then it can be used to either identify (already imported) dependencies to be built, or to build the app container, either locally or remotely. --- diff --git a/flatpak_module_tools/rpm_builder.py b/flatpak_module_tools/rpm_builder.py index c26d83a..8e4252e 100644 --- a/flatpak_module_tools/rpm_builder.py +++ b/flatpak_module_tools/rpm_builder.py @@ -197,7 +197,10 @@ class RpmBuilder: source_tag_infos = source_session.listTagged( source_tag, latest=True, inherit=True, package=package ) - source_version_info = StrippedVersionInfo.from_dict(source_tag_infos[0]) + if source_tag_infos: + source_version_info = StrippedVersionInfo.from_dict(source_tag_infos[0]) + else: + source_version_info = None ok = False @@ -206,7 +209,7 @@ class RpmBuilder: ) if package_tag_infos: package_version_info = StrippedVersionInfo.from_dict(package_tag_infos[0]) - if allow_outdated or package_version_info >= source_version_info: + if allow_outdated or not source_version_info or package_version_info >= source_version_info: ok = True create_event = package_tag_infos[0]["create_event"] @@ -218,13 +221,13 @@ class RpmBuilder: local_version_info = localrepo_package_versions.get(package) if local_version_info: - if allow_outdated or local_version_info >= source_version_info: + if allow_outdated or not source_version_info or local_version_info >= source_version_info: ok = True display_table.append(( ok, package, - str(source_version_info), + str(source_version_info) if source_version_info else "", str(package_version_info) if package_version_info else "", str(local_version_info) if local_version_info else "", "(forced)" if package in manual_packages else "", From dbfc453d008b216ca4d338335eeb16e94fdfe675 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 10/16] RpmBuilder: allow local repo builds of packages not yet in Koji This allows build-rpm-local to work on one or more source directories even if the package has not yet been imported into Koji, or hasn't yet reached the configured source_koji_tag. In this case, the user is completely responsible for dependency ordering by batching the commands accordingly; all directories passed will be built in parallel. --- diff --git a/flatpak_module_tools/rpm_builder.py b/flatpak_module_tools/rpm_builder.py index 8e4252e..f7dae04 100644 --- a/flatpak_module_tools/rpm_builder.py +++ b/flatpak_module_tools/rpm_builder.py @@ -494,9 +494,19 @@ class RpmBuilder: if not to_build: return - latest_builds = self._get_latest_builds(to_build) - - build_after = self._compute_build_order(latest_builds, include_localrepo=True) + try: + latest_builds = self._get_latest_builds(to_build) + build_after = self._compute_build_order(latest_builds, include_localrepo=True) + except Exception as e: + if not manual_packages and not auto: + # all packages are manual_repos, just build what has been passed, + # without taking responsibility for the build order + latest_builds = repo_map + build_after = { + repo.name: set() for repo in manual_repos + } + else: + raise e mock_cfg = make_mock_cfg( arch=self.arch, From c680214d15ca330c6e96936a723d7b8529025da7 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 11/16] generator: set runtime-name based on the flathub runtime used If org.kde.Platform is used on Flathub, we will generally want to use the matching flatpak-kdeN-runtime in Fedora. flathub-runtime remains the default and overall fallback. --- diff --git a/flatpak_module_tools/flatpak_generator.py b/flatpak_module_tools/flatpak_generator.py index a18e88a..ad0c278 100644 --- a/flatpak_module_tools/flatpak_generator.py +++ b/flatpak_module_tools/flatpak_generator.py @@ -101,6 +101,16 @@ class FlatpakGenerator(str): app_id = manifest.get('app-id') if app_id is None: app_id = manifest['id'] + if runtime_name is None: + flathub_runtime = manifest['runtime'] + flathub_runtime_version = manifest['runtime-version'] + if flathub_runtime == 'org.kde.Platform' and flathub_runtime_version.split('.')[0] == '6': + runtime_name = 'flatpak-kde6-runtime' + elif flathub_runtime == 'org.kde.Platform' and flathub_runtime_version.split('.')[0] == '5': + runtime_name = 'flatpak-kde5-runtime' + else: + runtime_name = 'flatpak-runtime' + yml = NoSortMapping({ 'flatpak': NoSortMapping({ 'id': app_id, @@ -136,6 +146,9 @@ class FlatpakGenerator(str): command = pkg branch = 'f' + str(runtime_version) + if runtime_name is None: + runtime_name = 'flatpak-runtime' + container_yaml = dedent(f'''\ flatpak: # Derived from the project's domain name @@ -184,9 +197,6 @@ class FlatpakGenerator(str): raise click.ClickException(f"{output_containerspec} exists." f" Pass --force to overwrite.") - if runtime_name is None: - runtime_name = 'flatpak-runtime' - if runtime_version is None: response = requests.get("https://bodhi.fedoraproject.org/releases/?state=current") response.raise_for_status() From 492299f0b3d13b83171f5e33cf1d39c9d8996326 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 12/16] Fix test_rpm_utils with RPM 4.20 Support for "the experimental non-privileged chroot support" was removed in RPM 4.20 (Fedora 41). Running the command in unshare is a workaround. https://github.com/rpm-software-management/rpm/issues/3441 https://github.com/rpm-software-management/rpm/pull/2668 --- diff --git a/tests/test_rpm_utils.py b/tests/test_rpm_utils.py index f9c983f..234ea4d 100644 --- a/tests/test_rpm_utils.py +++ b/tests/test_rpm_utils.py @@ -59,7 +59,7 @@ def rpmroot(tmp_path_factory): ) subprocess.check_call([ - "rpm", "--root", root, "-Uvh", testrpm, testrpm_epoch, testrpm_usr + "unshare", "--mount", "--user", "--map-root-user", "rpm", "--root", root, "-Uvh", testrpm, testrpm_epoch, testrpm_usr ]) # We don't necessarily expect GPG keys to be imported into the roots we @@ -69,7 +69,7 @@ def rpmroot(tmp_path_factory): f.write(FEDORA_GPG_KEY_RAWHIDE_X86_64) subprocess.check_call([ - "rpm", "--root", root, "--import", gpg_key + "unshare", "--mount", "--user", "--map-root-user", "rpm", "--root", root, "--import", gpg_key ]) return root From 89069df014b3b46ee92ad268dfe4220fc01f57fa Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 13/16] Drop pkg_resources usage pkg_resources.resource_stream is deprecated in favour of importlib.resources. This should be compatible with Python 3.7 and newer. https://setuptools.pypa.io/en/latest/pkg_resources.html https://docs.python.org/3.13/library/importlib.resources.html#module-importlib.resources --- diff --git a/flatpak-module-tools.spec b/flatpak-module-tools.spec index ad241f4..378c5bb 100644 --- a/flatpak-module-tools.spec +++ b/flatpak-module-tools.spec @@ -46,8 +46,6 @@ Requires: python3-jinja2 Requires: python3-koji Requires: python3-networkx Requires: python3-requests-toolbelt -# for pkg_resources -Requires: python3-setuptools Requires: python3-solv %description diff --git a/flatpak_module_tools/config.py b/flatpak_module_tools/config.py index fb1bdb2..64f1e14 100644 --- a/flatpak_module_tools/config.py +++ b/flatpak_module_tools/config.py @@ -2,8 +2,8 @@ from functools import cached_property import os from typing import Optional +import importlib.resources import koji -import pkg_resources import re import yaml @@ -168,7 +168,7 @@ class Config: def read(self): default_config_file = \ - pkg_resources.resource_stream('flatpak_module_tools', 'config.yaml') + importlib.resources.open_binary('flatpak_module_tools', 'config.yaml') self._read_config_file(default_config_file) for config_file in self._iter_config_files(): diff --git a/pyproject.toml b/pyproject.toml index 06140b2..eb7c7b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ cli = [ "koji", "networkx", "requests-toolbelt", - "setuptools", "solv", ] @@ -46,7 +45,6 @@ tests = [ "pytest-cov", "responses", "rpm", - "setuptools", ] [tool.setuptools] From f1b952dfe9a1ba48a288f659a00b9a3b996e02de Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 14/16] tests: register custom markers This avoids several PytestUnknownMarkWarning in test_cli and test_flatpak_generator. https://docs.pytest.org/en/stable/how-to/mark.html#registering-marks --- diff --git a/pyproject.toml b/pyproject.toml index eb7c7b5..4a2a7cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,10 @@ exclude_also = [ [tool.pytest.ini_options] addopts = "--cov=flatpak_module_tools --cov-report=term-missing --cov-report=html" +markers = [ + "container_yaml", + "needs_metadata" +] testpaths = [ "tests", ] From 7eff6dd08e2c6f8e03ac342eb91bb762c629710f Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 15/16] tests: avoid deprecated Modulemd methods modulemd_module_stream_read_string() is deprecated since 2.11 in favour of modulemd_read_packager_string(): https://fedora-modularity.github.io/libmodulemd/latest/ModulemdModuleStream.html#modulemd-module-stream-read-string https://fedora-modularity.github.io/libmodulemd/latest/modulemd-2.0-Modulemd.html#modulemd-read-packager-string --- diff --git a/tests/test_flatpak_builder.py b/tests/test_flatpak_builder.py index 627991c..6bcb3f4 100644 --- a/tests/test_flatpak_builder.py +++ b/tests/test_flatpak_builder.py @@ -175,7 +175,7 @@ def R(arch): @pytest.fixture def runtime_module(R): - runtime_mmd = Modulemd.ModuleStream.read_string(FLATPAK_RUNTIME_MMD, True) + runtime_mmd = Modulemd.read_packager_string(FLATPAK_RUNTIME_MMD) yield ModuleInfo(runtime_mmd.get_module_name(), runtime_mmd.get_stream_name(), runtime_mmd.get_version(), @@ -188,7 +188,7 @@ def runtime_module(R): def testapp_module(arch, R): testapp_mmd = TESTAPP_MMD.replace("@ARCH@", arch.rpm) - testapp_mmd = Modulemd.ModuleStream.read_string(testapp_mmd, True) + testapp_mmd = Modulemd.read_packager_string(testapp_mmd) yield ModuleInfo(testapp_mmd.get_module_name(), testapp_mmd.get_stream_name(), testapp_mmd.get_version(), From 2cc8a83af2efc630de3e47e3d584db4be0be5a8f Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Jan 21 2025 17:00:18 +0000 Subject: [PATCH 16/16] tests: avoid deprecated responses syntax The `stream` argument to `CallbackResponse` has been deprecated since 0.15 in favour of adding `stream` as an argument in the request: https://github.com/getsentry/responses/blob/master/README.rst#deprecations-and-migration-path https://github.com/getsentry/responses/blob/master/README.rst#request-keyword-arguments-matcher --- diff --git a/tests/test_package_locator.py b/tests/test_package_locator.py index 2901eac..8188775 100644 --- a/tests/test_package_locator.py +++ b/tests/test_package_locator.py @@ -112,7 +112,7 @@ class StreamingCompressedResponse(responses.CallbackResponse): } return (200, headers, BufferedReader(RawBytesReader(compressed_data))) - super().__init__(method=method, url=url, callback=callback, stream=True, **kwargs) + super().__init__(method=method, url=url, callback=callback, **kwargs) @pytest.mark.parametrize("compress_type,extension", [ @@ -141,7 +141,8 @@ def test_package_locator(compress_type, extension): responses.GET, f"https://repos.example.com/basic/ppc64le/repodata/HASH-primary.xml{extension}", body_str=BASIC_PRIMARY_XML, - compress_type=compress_type + compress_type=compress_type, + match=[responses.matchers.request_kwargs_matcher({"stream": True})] )) # basic operation - find highest version