From e94dc065666bd7ff4940c585747194400d915d9a Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Jan 13 2022 11:31:09 +0000 Subject: Support building SRPMs in target mock New arguments for building source RPMs are introduced: * x-pkg srpm --srpm-mock: Creates source RPM with 'mock' instead of 'rpmbuild' * x-pkg mockbuild --srpm-mock: Generate source rpm with 'mock' and then run 'mockbuild' using this source rpm. * x-pkg build --srpm-mock: Use source rpm generated with mock for building. * x-pkg scratch-build --srpm-mock It works for projects with single specfile. JIRA: RHELCMP-374 Fixes: #495 Signed-off-by: Ondrej Nosek --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 20ce644..879a42e 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -663,8 +663,29 @@ class Commands(object): self.load_nameverrel() return self._uses_rpmautospec + def _parse_output_for_nameverrel(self, output): + """Parses an output of the 'rpm' command - extracts fields from + the string and set the Command object's properties. + """ + # Get just the output, then split it by ??, grab the first and split + # again to get ver and rel + first_line_output = output.split('??')[1] + parts = first_line_output.split() + if len(parts) != 4: + raise rpkgError('Could not get n-v-r-e from %r' + % first_line_output) + (self._package_name_spec, + self._epoch, + self._ver, + self._rel) = parts + + # Most packages don't include a "Epoch: 0" line, in which case RPM + # returns '(none)' + if self._epoch == "(none)": + self._epoch = "0" + def load_nameverrel(self): - """Set the release of a package.""" + """Set the release and version of a package.""" # If the repo is a container, we check for 'verrel' information in the Dockerfile. if self.ns in ("container", "containers"): @@ -723,22 +744,93 @@ class Commands(object): raise rpkgError('Could not get n-v-r-e from %s' % os.path.join(self.layout.specdir, self.spec)) - # Get just the output, then split it by ??, grab the first and split - # again to get ver and rel - first_line_output = output.split('??')[1] - parts = first_line_output.split() - if len(parts) != 4: - raise rpkgError('Could not get n-v-r-e from %r' - % first_line_output) - (self._package_name_spec, - self._epoch, - self._ver, - self._rel) = parts + self._parse_output_for_nameverrel(output) - # Most packages don't include a "Epoch: 0" line, in which case RPM - # returns '(none)' - if self._epoch == "(none)": - self._epoch = "0" + def load_nameverrel_mock(self, mockargs=[], root=None, force_local_mock_config=None): + """Set the release and version of a package with mock. + + Log the output and returns nothing + + :param list mockargs: list of command line arguments which are passed + mock. + :param str root: chroot config name which is passed to mock ``-r`` + option. + :param bool force_local_mock_config: enforce download of the Mock + configuration from Kojihub (True), or enforce local Mock config + (False). If None local configuration is used if present. + + .. versionadded:: 1.64 + """ + + # Otherwise, we get 'verrel' information from the '.spec' file. + rpm_cmd = ['rpm'] + rpm_cmd.extend(self.rpmdefines) + + specfile_path = os.path.join(self.layout.specdir, self.spec) + + if specfile_uses_rpmautospec: + self._uses_autorelease = specfile_uses_rpmautospec( + specfile_path, check_autorelease=True, check_autochangelog=False + ) + self._uses_rpmautospec = specfile_uses_rpmautospec(specfile_path) + if self._uses_rpmautospec and rpmautospec_calculate_release_number: + release_number = rpmautospec_calculate_release_number(specfile_path) + rpm_cmd.append("--define '_rpmautospec_release_number %d'" % release_number) + else: + # Set to 0 so it evaluates false-ish but differs from (unset) None. + self._uses_autorelease = 0 + self._uses_rpmautospec = 0 + + # setup the command + cmd = ['mock'] + cmd.extend(mockargs) + if self.quiet: + cmd.append('--quiet') + + root, config_dir = self.get_mock_config_dir(root, force_local_mock_config) + if config_dir: + cmd.extend(['--configdir', config_dir]) + + # Temporary directory for results (rpm command output) and mock's logs + # (root.log, state.log, ...). These logs are not used. + tmp_resultdir = tempfile.mkdtemp(prefix="mock_resultdir") + cmd += ['-r', root, '--chroot', '--resultdir', tmp_resultdir] + + tmp_root = '/var/tmp' # temporary directory inside the mock root + copyin_cmd = cmd + ['--copyin', specfile_path, tmp_root] + + # We make sure there is a space at the end of our query so that + # we can split it later. When there are subpackages, we get a + # listing for each subpackage. We only care about the first. + rpm_cmd.extend(['-q', '--qf', '"??%{NAME} %{EPOCH} %{VERSION} %{RELEASE}??"', + '--specfile', '"%s"' % os.path.join(tmp_root, self.spec)]) + main_cmd = cmd + ['--shell'] + [' '.join(rpm_cmd)] \ + + ['> ' + os.path.join(tmp_root, 'output')] + + copyout_cmd = cmd + ['--copyout', os.path.join(tmp_root, 'output'), tmp_resultdir] + + try: + # 'copyin' copies specfile into mock's root directory to be accessible for the rpm cmd + self.log.debug('Running "copyin" command: %s' % ' '.join(copyin_cmd)) + self._run_command(copyin_cmd) + # the main command runs 'rpm' and its output redirects into the file + self.log.debug('Running "rpm" command: %s' % ' '.join(main_cmd)) + self._run_command(main_cmd) + # 'copyout' copies result outside of mock's root directory to be parsed later + self.log.debug('Running "copyout" command: %s' % ' '.join(copyout_cmd)) + self._run_command(copyout_cmd) + with open(os.path.join(tmp_resultdir, 'output'), 'r') as f: + output = f.read() + except Exception as e: + self.log.debug('Errors occoured while running above command to get N-V-R-E in mock.') + raise rpkgError('Could not query n-v-r of %s: %s' + % (self.repo_name, e)) + finally: + self.log.debug('Cleaning up mock temporary config directory: %s', config_dir) + self._cleanup_tmp_dir(config_dir) + self._cleanup_tmp_dir(tmp_resultdir) + + self._parse_output_for_nameverrel(output) @property def repo(self): @@ -2844,8 +2936,28 @@ class Commands(object): return True return False + def get_mock_config_dir(self, root=None, force_local_mock_config=None): + config_dir = None + if not root: + root = self.mockconfig + if not self.use_local_mock_config(root, force_local_mock_config): + self.log.debug('Going to download Mock config from Kojihub') + try: + config_dir = self._config_dir_basic(root=root) + except rpkgError as error: + raise rpkgError('Failed to create mock config directory:' + ' %s' % error) + self.log.debug('Temporary mock config directory: %s', config_dir) + try: + self._config_dir_other(config_dir) + except rpkgError as error: + self._cleanup_tmp_dir(config_dir) + raise rpkgError('Failed to populate mock config directory:' + ' %s' % error) + return root, config_dir + def mockbuild(self, mockargs=[], root=None, hashtype=None, shell=None, - force_local_mock_config=None): + force_local_mock_config=None, srpm_mock=False): """Build the package in mock, using mockargs Log the output and returns nothing @@ -2857,16 +2969,23 @@ class Commands(object): :param str hashtype: used to generate SRPM only if there is no SRPM generated before. :param bool shell: indicate whether to go into chroot. - :param bool koji_config: enforce download of the Mock configuration - from Kojihub (True), or enforce local Mock config (False). If - None local configuration is used if present. + :param bool force_local_mock_config: enforce download of the Mock + configuration from Kojihub (True), or enforce local Mock config + (False). If None local configuration is used if present. + :param bool srpm_mock: genereate source rpm .. versionadded:: 1.56 Parameter shell. + .. versionadded:: 1.64 + Parameter srpm_mock. """ - - # Make sure we have an srpm to run on - self.srpm(hashtype=hashtype) + # Make sure we have an srpm to run on if we need it + if srpm_mock or (hasattr(self, 'srpmname') and self.srpmname): + # srpm is being generated or mockbuild is running with path + # to already generated srpm + pass + else: + self.srpm(hashtype=hashtype) # setup the command cmd = ['mock'] @@ -2874,29 +2993,16 @@ class Commands(object): if self.quiet: cmd.append('--quiet') - config_dir = None - if not root: - root = self.mockconfig - if not self.use_local_mock_config(root, force_local_mock_config): - self.log.debug('Going to download Mock config from Kojihub') - try: - config_dir = self._config_dir_basic(root=root) - except rpkgError as error: - raise rpkgError('Failed to create mock config directory:' - ' %s' % error) - self.log.debug('Temporary mock config directory: %s', config_dir) - try: - self._config_dir_other(config_dir) - except rpkgError as error: - self._cleanup_tmp_dir(config_dir) - raise rpkgError('Failed to populate mock config directory:' - ' %s' % error) - cmd.extend(['--configdir', config_dir]) + root, config_dir = self.get_mock_config_dir(root, force_local_mock_config) + if config_dir: + cmd.extend(['--configdir', config_dir]) cmd += ['-r', root, '--resultdir', self.mock_results_dir] if shell: cmd.append('--shell') + elif srpm_mock: + cmd += ['--buildsrpm', '--sources', self.layout.sourcedir, '--spec', self.spec] else: cmd += ['--rebuild', self.srpmname] diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py index a2cdc6d..da563b8 100644 --- a/pyrpkg/cli.py +++ b/pyrpkg/cli.py @@ -546,6 +546,10 @@ class cliClient(object): '--srpm', nargs='?', const='CONSTRUCT', help='Build from an srpm. If no srpm is provided with this option' ' an srpm will be generated from current module content.') + build_parser.add_argument( + '--srpm-mock', action='store_true', + help='Build from an srpm. Source rpm will be generated in \'mock\'' + ' instead of \'rpmbuild\'.') build_parser.set_defaults(command=self.build) def register_chainbuild(self): @@ -1115,6 +1119,10 @@ class cliClient(object): mockbuild_parser.add_argument( '--enable-network', action='store_true', help='Enable networking') mockbuild_parser.add_argument( + "--srpm-mock", action='store_true', + help='Generate source rpm with \'mock\' and then run \'mockbuild\' ' + 'using this source rpm') + mockbuild_parser.add_argument( "extra_args", default=None, nargs=argparse.REMAINDER, help="Custom arguments that are passed to the 'mock'. " "Use '--' to separate them from other arguments.") @@ -1457,6 +1465,10 @@ class cliClient(object): help='Build from an srpm. If no srpm is provided with this ' 'option an srpm will be generated from the current module ' 'content.') + scratch_build_parser.add_argument( + '--srpm-mock', action='store_true', + help='Build from an srpm. Source rpm will be generated in \'mock\'' + ' instead of \'rpmbuild\'.') scratch_build_parser.set_defaults(command=self.scratch_build) def register_sources(self): @@ -1496,6 +1508,19 @@ class cliClient(object): srpm_parser.add_argument( '--md5', action='store_const', const='md5', default=None, dest='hash', help='Use md5 checksums (for older rpm hosts)') + srpm_parser.add_argument( + '--srpm-mock', action='store_true', + help='Create source rpm in \'mock\' instead of \'rpmbuild\'') + srpm_parser.add_argument( + '--no-clean', '-n', help='Only for --srpm-mock: Do not clean ' + 'chroot before building package', action='store_true') + srpm_parser.add_argument( + '--no-cleanup-after', help='Only for --srpm-mock: Do not clean ' + 'chroot after building if automatic cleanup is enabled', + action='store_true') + srpm_parser.add_argument( + '--no-clean-all', '-N', help='Only for --srpm-mock: Alias for ' + 'both --no-clean and --no-cleanup-after', action='store_true') srpm_parser.set_defaults(command=self.srpm) def register_copr_build(self): @@ -1821,6 +1846,26 @@ class cliClient(object): uploaded and None is returned. :rtype: str """ + if hasattr(self.args, 'srpm_mock') and self.args.srpm_mock: + # Set the release and version of a package with mock. Mockbuild needs them. + self.cmd.load_nameverrel_mock(mockargs=tuple(), + root=None, + force_local_mock_config=None) + # generate srpm with mock instead of rpmbuild + self.log.debug('Generating an srpm with mock') + self.cmd.mockbuild(mockargs=tuple(), + root=None, + hashtype=self.args.hash, + shell=None, + force_local_mock_config=None, + srpm_mock=True) + # get newly generated source rpm path for upload + srpmname = os.path.join(self.cmd.mock_results_dir, + "%s-%s-%s.src.rpm" + % (self.cmd.repo_name, self.cmd.ver, self.cmd.rel)) + self.log.debug('Srpm generated: {0}'.format(srpmname)) + return self._upload_file_for_build(srpmname) + if hasattr(self.args, 'srpm') and self.args.srpm: # See if we need to generate the srpm first if self.args.srpm == 'CONSTRUCT': @@ -2319,10 +2364,25 @@ class cliClient(object): # there were no args pass try: - self.cmd.mockbuild( - mockargs, self.args.root, hashtype=self.args.hash, - shell=self.args.shell, - force_local_mock_config=self.args.local_mock_config) + if self.args.srpm_mock: + # Set the release and version of a package with mock. Mockbuild needs them. + self.cmd.load_nameverrel_mock(mockargs, self.args.root, + force_local_mock_config=self.args.local_mock_config) + self.log.debug('Generating an srpm with mock') + self.cmd.mockbuild(mockargs, self.args.root, + hashtype=self.args.hash, + shell=self.args.shell, + force_local_mock_config=self.args.local_mock_config, + srpm_mock=True) + # pass newly generated source rpm path to mockbuild + self.cmd.srpmname = os.path.join(self.cmd.mock_results_dir, + "%s-%s-%s.src.rpm" + % (self.cmd.repo_name, self.cmd.ver, self.cmd.rel)) + self.log.debug('Srpm generated: {0}'.format(self.cmd.srpmname)) + self.cmd.mockbuild(mockargs, self.args.root, + hashtype=self.args.hash, + shell=self.args.shell, + force_local_mock_config=self.args.local_mock_config) except Exception as e: raise rpkgError(e) @@ -2723,16 +2783,39 @@ class cliClient(object): def srpm(self): self.sources() - # Koji does not allow defines, custom builddir and custom buildroot. - # Argparse won't set them for koji style builds. Normal koji builds - # are for all arches and not set via argparse. - self.cmd.srpm( - builddir=getattr(self.args, 'builddir', None), - arch=getattr(self.args, 'arch', None), - define=getattr(self.args, 'define', None), - extra_args=self.extra_args, - buildrootdir=getattr(self.args, 'buildrootdir', None), - hashtype=self.args.hash,) + if hasattr(self.args, 'srpm_mock') and self.args.srpm_mock: + mockargs = [] + + if (hasattr(self.args, 'no_clean') and self.args.no_clean) \ + or (hasattr(self.args, 'no_clean_all') or self.args.no_clean_all): + mockargs.append('--no-clean') + + if (hasattr(self.args, 'no_cleanup_after') and self.args.no_cleanup_after) \ + or (hasattr(self.args, 'no_clean_all') and self.args.no_clean_all): + mockargs.append('--no-cleanup-after') + + # Set the release and version of a package with mock. Mockbuild needs them. + self.cmd.load_nameverrel_mock(mockargs=tuple(), + root=None, + force_local_mock_config=None) + # generate srpm with mock instead of rpmbuild + self.cmd.mockbuild(mockargs=tuple(), + root=None, + hashtype=self.args.hash, + shell=None, + force_local_mock_config=None, + srpm_mock=True) + else: + # Koji does not allow defines, custom builddir and custom buildroot. + # Argparse won't set them for koji style builds. Normal koji builds + # are for all arches and not set via argparse. + self.cmd.srpm( + builddir=getattr(self.args, 'builddir', None), + arch=getattr(self.args, 'arch', None), + define=getattr(self.args, 'define', None), + extra_args=self.extra_args, + buildrootdir=getattr(self.args, 'buildrootdir', None), + hashtype=self.args.hash,) def switch_branch(self): if self.args.branch: