From 575b04fe7ce5a6486619b608bb5abef31d14a849 Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 1/11] Plugin taskrepos Fixes: https://pagure.io/koji/issue/3871 --- diff --git a/plugins/builder/taskrepos.conf b/plugins/builder/taskrepos.conf new file mode 100644 index 0000000..b75c69b --- /dev/null +++ b/plugins/builder/taskrepos.conf @@ -0,0 +1,4 @@ +[taskrepos] +# expire_repos = 14 +# ticketlink = +# sourcecodelink = https://pagure.io/koji/blob/master/f/plugins/builder/taskrepos.py diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py new file mode 100644 index 0000000..aabd0bd --- /dev/null +++ b/plugins/builder/taskrepos.py @@ -0,0 +1,366 @@ +# kojid plugin + +from __future__ import absolute_import + +import datetime +import fcntl +import os +import six +import smtplib +import subprocess + +import koji +from koji import request_with_retry +import koji.tasks +from koji.tasks import ( + BaseTaskHandler +) +from koji.util import ( + joinpath, + rmtree +) + +__all__ = ('TaskReposTask',) + +CONFIG_FILE = '/etc/kojid/plugins/taskrepos.conf' + + +class TaskReposTask(koji.tasks.BaseTaskHandler): + + Methods = ['taskrepos'] + _taskWeight = 2.0 + + def __init__(self, *args, **kwargs): + self._read_config() + return super(TaskReposTask, self).__init__(*args, **kwargs) + + def _read_config(self): + cp = koji.read_config_files(CONFIG_FILE) + self.config = { + 'REPO_LIFETIME': 14, # days + } + + # expire repos in days + if cp.has_option('taskrepos', 'expire_repos'): + self.config['REPO_LIFETIME'] = cp.get('taskrepos', 'expire_repos') + + def rm_contents(self, path): + with os.scandir(path) as it: + for e in it: + if e.is_dir(follow_symlinks=False): + rmtree(e.path) + else: + os.remove(e.path) + + def requests_get(self, url, filedest): + self.logger.debug(f'Retrieving {url} to {filedest}...') + start = datetime.datetime.utcnow() + resp = request_with_retry().get(url, stream=True) + try: + with open(filedest, 'wb') as fo: + for chunk in resp.iter_content(chunk_size=1048576): + fo.write(chunk) + finally: + resp.close() + end = datetime.datetime.utcnow() + self.logger.debug(f'Received {os.path.getsize(filedest)} bytes in {end - start}') + + def mirror_rpms(self, rpms, content_dir, build=None): + remote_pi = koji.PathInfo(topdir=self.options.topurl) + for rpm in rpms: + if build: + # rpm is a dict of rpminfo + url = remote_pi.build(build) + '/' + remote_pi.rpm(rpm) + dest = joinpath(content_dir, rpm['arch']) + rpmname = f"{rpm['nvr']}.{rpm['arch']}.rpm" + else: + # rpm is a relative path under the work/ directory + taskdir, rpmname = rpm.split('/')[-2:] + url = remote_pi.work() + '/' + rpm + dest = joinpath(content_dir, taskdir) + koji.ensuredir(dest) + filedest = joinpath(dest, rpmname) + self.requests_get(url, filedest) + + def gen_repodata(self, rpmdir, baseurl): + cmd = [ + '/usr/bin/createrepo_c', + '--database', + '--checksum=sha256', + f'--baseurl={baseurl}', + '--general-compress-type=gz', + rpmdir, + ] + self.logger.debug(f'Running: {" ".join(cmd)}') + + proc = subprocess.run(cmd, cwd=rpmdir, capture_output=True) + self.logger.debug(proc.stdout.decode()) + if proc.stderr: + self.logger.debug(proc.stderr.decode()) + proc.check_returncode() + + def merge_arch_repo(self, arch, repodir, srcdir): + archdir = joinpath(repodir, arch) + self.logger.debug(f'Creating {arch} repo under {archdir}') + koji.ensuredir(archdir) + cmd = [ + '/usr/bin/mergerepo_c', + '--koji', + '--database', + f'--outputdir={archdir}', + f'--archlist={arch},noarch,src', + '--compress-type=gz', + ] + for srcrepo in os.listdir(srcdir): + cmd.append(f'--repo={srcrepo}') + self.logger.debug(f'Running: {" ".join(cmd)}') + proc = subprocess.run(cmd, cwd=srcdir, capture_output=True) + self.logger.debug(proc.stdout.decode()) + if proc.stderr: + self.logger.debug(proc.stderr.decode()) + proc.check_returncode() + + def build_arches(self, rpms): + arches = set() + for rpm in rpms: + if isinstance(rpm, str): + rpm = koji.parse_NVRA(os.path.basename(rpm)) + if rpm['arch'] == 'src': + continue + arches.add(rpm['arch']) + return arches + + def create_repos(self, taskinfo): + builds = self.session.listBuilds(taskID=taskinfo['id']) + build = None + task_id = str(self.id) + if builds: + build = builds[0] + rpms = self.session.listRPMs(buildID=build['build_id']) + repodir = joinpath(self.workdir, 'taskrepos', task_id, task_id, + build['name'], build['version'], build['release']) + nvr = f"{build['name']}.{build['version']}.{build['release']}" + else: + children = self.session.getTaskChildren(taskinfo['id'], request=True) + srpminfo = None + rpms = [] + for child in children: + if child['method'] != 'buildArch': + continue + results = self.session.getTaskResult(child['id']) + if results.get('srpms'): + rpms.append(results['srpms'][0]) + srpmname = os.path.basename(results['srpms'][0]) + srpminfo = koji.parse_NVRA(srpmname) + rpms.extend(results['rpms']) + assert srpminfo, 'missing srpm' + repodir = joinpath(self.workdir, 'taskrepos', task_id, task_id, 'scratch', + srpminfo['name'], srpminfo['version'], srpminfo['release']) + nvr = f"{srpminfo['name']}.{srpminfo['version']}.{srpminfo['release']}" + koji.ensuredir(repodir) + + dirfd = os.open(repodir, os.O_RDONLY) + fcntl.flock(dirfd, fcntl.LOCK_EX) + self.rm_contents(repodir) + content_dir = joinpath(repodir, '_content') + self.mirror_rpms(rpms, content_dir, build=build) + pi = koji.PathInfo(self.options.topurl) + for subdir in os.listdir(content_dir): + if build: + baseurl = pi.build(build) + '/' + subdir + else: + baseurl = pi.task(int(subdir)) + self.gen_repodata(joinpath(content_dir, subdir), baseurl) + for arch in self.build_arches(rpms): + self.merge_arch_repo(arch, repodir, content_dir) + rmtree(content_dir) + return repodir, baseurl, nvr + + def link_task_repos(self, task_id, repodir): + """Symlink repodir to $basedir/tasks/{task_id}""" + tasksdir = joinpath(self.workdir, 'tasks') + if not os.path.exists(tasksdir): + koji.ensuredir(tasksdir) + dstdir = joinpath(tasksdir, str(task_id)) + if os.path.islink(dstdir) or os.path.isfile(dstdir): + os.remove(dstdir) + elif os.path.isdir(dstdir): + rmtree(dstdir) + os.symlink(os.path.relpath(repodir, tasksdir), dstdir) + + def write_repo_file(self, task, repodir): + nvr = '-'.join(repodir.split('/')[-3:]) + if len(task['request']) > 2: + scratch = task['request'][2].get('scratch', False) + else: + scratch = False + subtasks = [] + if task['method'] == 'createrepo': + subtasks = [subtask for subtask in task.getChildren(task['id'], request=True)] + repo_arches = [] + for subtask in subtasks: + repo_arches.append(f'{self.options.topurl}/{subtask["label"]}') + if 'noarch' in repo_arches: + repo_url = f'{self.options.topurl}/noarch/' + else: + repo_url = f'{self.options.topurl}/$basearch/' + repo_file_name = nvr + repo_name = f'brew-task-repo-{nvr.replace('+', '_')}' + build = 'build' + if scratch: + repo_file_name += '-scratch' + repo_name += '-scratch' + build = 'scratch build' + repo_file_name += '.repo' + repo_file = f'{repodir}/{repo_file_name}' + + with koji._open_text_file(repo_file, 'w') as repo_fd: + repo_fd.write( + f"""[{repo_name}] + name=Repo for Brew {build} of {nvr} + enabled=1 + gpgcheck=0 + baseurl={repo_url} + module_hotfixes=1 + """ + ) + self.logger.debug(f'Wrote repo file to {repo_file}') + return repo_file_name + + def upload_repo(self, repodir): + repo_files = [] + repo_paths = [] + uploadpath_basic = "taskrepos/%s/%s" % (self.id % 10000, self.id) + for dirpath, dirs, files in os.walk(repodir): + relrepodir = os.path.relpath(dirpath, repodir) + for filename in files: + path = "%s/%s" % (dirpath, filename) + if os.path.islink(path): + continue + relpath = "%s/%s" % (relrepodir, filename) + localpath = '%s/%s' % (repodir, relpath) + reldir = os.path.dirname(relpath) + if reldir: + uploadpath = "%s/%s" % (uploadpath_basic, reldir) + fn = os.path.basename(relpath) + else: + uploadpath = uploadpath_basic + fn = relpath + self.session.uploadWrapper(localpath, uploadpath, fn) + repo_paths.append(joinpath(uploadpath, filename)) + repo_files.append(relpath) + return uploadpath_basic, repo_files, repo_paths + + def handler(self, task_id_child, task_link=False): + if not isinstance(task_id_child, int): + task_id_child = int(task_id_child) + taskinfo = self.session.getTaskInfo(task_id_child, request=True) + policy_data = { + 'user_id': taskinfo['owner'], + } + self.session.host.assertPolicy('taskrepos', policy_data) + if not taskinfo: + raise koji.BuildError(f'Invalid task ID: {task_id_child}') + if taskinfo['method'] != 'build': + raise koji.BuildError(f'{task_id_child} is not a build task') + build_subtask = next(task for task in self.session.getTaskChildren(task_id_child) + if task['method'] == 'buildArch') + if build_subtask['state'] != 2: + raise koji.BuildError(f'task {build_subtask['id']} has not completed successfully') + repodir, baseurl, nvr = self.create_repos(taskinfo) + self.logger.debug(f'Repos for task {task_id_child} created under {repodir}') + if task_link: + self.link_task_repos(task_id_child, repodir) + self.write_repo_file(taskinfo, repodir) + uploadpath, repo_files, repo_paths = self.upload_repo(repodir) + owner_id = taskinfo['owner'] + owner = self.session.getUser(owner_id)['name'] + self.session.tasksRepoNotifications(self.id, repodir, baseurl, owner, repo_paths, nvr) + return [uploadpath, repo_files] + + +class tasksRepoNotifications(BaseTaskHandler): + Methods = ['tasksRepoNotifications'] + + _taskWeight = 0.1 + + def __init__(self, *args, **kwargs): + self._read_config() + return super(tasksRepoNotifications, self).__init__(*args, **kwargs) + + def _read_config(self): + cp = koji.read_config_files(CONFIG_FILE) + self.config = { + 'REPO_LIFETIME': 14, # days + 'TICKETLINK': '', + 'SOURCECODE': '', + } + + # expire repos in days + if cp.has_option('taskrepos', 'expire_repos'): + self.config['REPO_LIFETIME'] = cp.get('taskrepos', 'expire_repos') + + if cp.has_option('taskrepos', 'ticketlink'): + self.config['TICKETLINK'] = cp.get('taskrepos', 'ticketlink') + + if cp.has_option('taskrepos', 'sourcecodelink'): + self.config['SOURCECODE'] = cp.get('taskrepos', 'sourcecodelink') + + # XXX externalize these templates somewhere + subject_templ = 'repo for %(build)s of %(nvr)s is available' + message_templ = \ + """From: "Brew Task Repos System" %(from_addr)s\r +Subject: %(subject)s\r +To: %(recipient)s\r +A yum repository for the %(build)s of %(nvr)s (task %(task_id)s) is available at:\r +\r +%(repodir)s/\r +\r +You can install the rpms locally by putting this .repo file in your /etc/yum.repos.d/ directory:\r +\r +%(repofile)s\r +\r +It is necessary to have internal Red Hat CA certificates installed on the system, if +you want to use yum repository. You can install required certificates from:\r +\r +* http://hdn.corp.redhat.com/rhel7-csb-stage/repoview/redhat-internal-cert-install.html (rpm)\r +* https://certs.corp.redhat.com/ (cert files)\r +\r +The full list of repos is:\r +%(repo_urls)s\r +\r +The repository will be available for the next %(repolifetime)s days. +Scratch build output will be deleted earlier, based on the Brew scratch build retention policy.\r +\r +If you found a bug or you wish to stop these emails, please create a ticket:\r +%(ticketlink)s\r +\r +Source code at %(sourcecode)s\r +""" + + def handler(self, recipient, repo_urls, task_id, repodir, repofile, nvr): + if len(recipient) == 0: + self.logger.debug(f'task {self.id}: no recipients, not sending notifications') + return + from_addr = self.options.from_addr + ticketlink = self.config['TICKETLINK'] + sourcecode = self.config['SOURCECODE'] + repolifetime = self.config['REPO_LIFETIME'] + build = nvr.split('.')[0] + repo_urls = '\n'.join(repo_urls) + subject = self.subject_templ % locals() + message = self.message_templ % locals() + + # ensure message is in UTF-8 + message = koji.fixEncoding(message) + # binary for python3 + if six.PY3: + message = message.encode('utf8') + server = smtplib.SMTP(self.options.smtphost) + if self.options.smtp_user is not None and self.options.smtp_pass is not None: + server.login(self.options.smtp_user, self.options.smtp_pass) + # server.set_debuglevel(True) + server.sendmail(from_addr, recipient, message) + server.quit() + + return f'sent notification of taskrepo {self.id} to: {recipient}' diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py new file mode 100644 index 0000000..2832d46 --- /dev/null +++ b/plugins/hub/taskrepos_hub.py @@ -0,0 +1,41 @@ +# koji hub plugin +# There is a kojid plugin that goes with this hub plugin. The kojid builder +# plugin has a config file. This hub plugin has no config file. + + +import kojihub + +from koji.context import context +from koji.plugin import export, callback +from koji.util import joinpath + +__all__ = ('taskrepos',) + + +@callback('postBuildStateChange') +def taskrepos(cbtype, *args, **kws): + task_id = kws['info']['task_id'] + task = kojihub.Task(task_id) + subtasks_method = [subtask['method'] for subtask in task.getChildren(task_id)] + if 'buildArch' not in subtasks_method: + return + task_link = False + + kojihub.make_task('taskrepos', [task_id, task_link]) + return + + +@export +def tasksRepoNotifications(task_id, repodir, baseurl, owner, repo_paths, nvr): + if context.opts.get('DisableNotifications'): + return + email_domain = context.opts['EmailDomain'] + email = f'{owner}@{email_domain}' + repo_urls = [] + for rf in repo_paths: + if '.repo' in rf: + repofile = joinpath(baseurl, rf) + else: + repo_urls.append(joinpath(baseurl, rf)) + kojihub.make_task('tasksRepoNotifications', + [email, repo_urls, task_id, repodir, repofile, nvr]) From 5ba5086ad58f508c12d4e9cbf7e2c1c3bcb7731b Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 2/11] Fix review --- diff --git a/plugins/builder/taskrepos.conf b/plugins/builder/taskrepos.conf index b75c69b..7769b2f 100644 --- a/plugins/builder/taskrepos.conf +++ b/plugins/builder/taskrepos.conf @@ -2,3 +2,27 @@ # expire_repos = 14 # ticketlink = # sourcecodelink = https://pagure.io/koji/blob/master/f/plugins/builder/taskrepos.py +email_template : From: %%(from_addr)s + Subject: repo for %%(build)s of %%(nvr)s is available + To: %%(recipient)s + A yum repository for the %%(build)s of %%(nvr)s (task %%(task_id)s) is available at: + %%(repodir)s/ + + You can install the rpms locally by putting this .repo file in your /etc/yum.repos.d/ directory: + %%(repofile)s + + It is necessary to have internal Red Hat CA certificates installed on the system, if + you want to use yum repository. You can install required certificates from: + * http://hdn.corp.redhat.com/rhel7-csb-stage/repoview/redhat-internal-cert-install.html (rpm) + * https://certs.corp.redhat.com/ (cert files) + + The full list of repos is: + %%(repo_urls)s + + The repository will be available for the next %%(repolifetime)s days. + Scratch build output will be deleted earlier, based on the Brew scratch build retention policy. + + If you found a bug or you wish to stop these emails, please create a ticket: + %%(ticketlink)s + + Source code at %%(sourcecode)s \ No newline at end of file diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index aabd0bd..6fc99ab 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -2,96 +2,65 @@ from __future__ import absolute_import -import datetime -import fcntl import os import six import smtplib import subprocess import koji -from koji import request_with_retry import koji.tasks from koji.tasks import ( BaseTaskHandler ) -from koji.util import ( - joinpath, - rmtree -) +from koji.util import joinpath + -__all__ = ('TaskReposTask',) +__all__ = ('TaskReposTask', 'tasksRepoNotifications') CONFIG_FILE = '/etc/kojid/plugins/taskrepos.conf' +config = None -class TaskReposTask(koji.tasks.BaseTaskHandler): +def read_config(): + global config + cp = koji.read_config_files(CONFIG_FILE) + config = { + 'repo_lifetime': 14, # days + 'ticketlink': '', + 'sourcecode': '', + 'email_template': '', + } - Methods = ['taskrepos'] - _taskWeight = 2.0 + # expire repos in days + if cp.has_option('taskrepos', 'expire_repos'): + config['repo_lifetime'] = cp.get('taskrepos', 'expire_repos') - def __init__(self, *args, **kwargs): - self._read_config() - return super(TaskReposTask, self).__init__(*args, **kwargs) + if cp.has_option('taskrepos', 'ticketlink'): + config['ticketlink'] = cp.get('taskrepos', 'ticketlink') - def _read_config(self): - cp = koji.read_config_files(CONFIG_FILE) - self.config = { - 'REPO_LIFETIME': 14, # days - } + if cp.has_option('taskrepos', 'sourcecodelink'): + config['sourcecode'] = cp.get('taskrepos', 'sourcecodelink') - # expire repos in days - if cp.has_option('taskrepos', 'expire_repos'): - self.config['REPO_LIFETIME'] = cp.get('taskrepos', 'expire_repos') - - def rm_contents(self, path): - with os.scandir(path) as it: - for e in it: - if e.is_dir(follow_symlinks=False): - rmtree(e.path) - else: - os.remove(e.path) + if cp.has_option('taskrepos', 'email_template'): + config['email_template'] = cp.get('taskrepos', 'email_template') - def requests_get(self, url, filedest): - self.logger.debug(f'Retrieving {url} to {filedest}...') - start = datetime.datetime.utcnow() - resp = request_with_retry().get(url, stream=True) - try: - with open(filedest, 'wb') as fo: - for chunk in resp.iter_content(chunk_size=1048576): - fo.write(chunk) - finally: - resp.close() - end = datetime.datetime.utcnow() - self.logger.debug(f'Received {os.path.getsize(filedest)} bytes in {end - start}') - - def mirror_rpms(self, rpms, content_dir, build=None): - remote_pi = koji.PathInfo(topdir=self.options.topurl) - for rpm in rpms: - if build: - # rpm is a dict of rpminfo - url = remote_pi.build(build) + '/' + remote_pi.rpm(rpm) - dest = joinpath(content_dir, rpm['arch']) - rpmname = f"{rpm['nvr']}.{rpm['arch']}.rpm" - else: - # rpm is a relative path under the work/ directory - taskdir, rpmname = rpm.split('/')[-2:] - url = remote_pi.work() + '/' + rpm - dest = joinpath(content_dir, taskdir) - koji.ensuredir(dest) - filedest = joinpath(dest, rpmname) - self.requests_get(url, filedest) + +class TaskReposTask(koji.tasks.BaseTaskHandler): + + Methods = ['taskrepos'] + _taskWeight = 2.0 def gen_repodata(self, rpmdir, baseurl): + koji.ensuredir(rpmdir) cmd = [ '/usr/bin/createrepo_c', '--database', + '--baseurl=%s' % baseurl, '--checksum=sha256', - f'--baseurl={baseurl}', '--general-compress-type=gz', rpmdir, ] - self.logger.debug(f'Running: {" ".join(cmd)}') + self.logger.debug('Running: %s' % " ".join(cmd)) proc = subprocess.run(cmd, cwd=rpmdir, capture_output=True) self.logger.debug(proc.stdout.decode()) @@ -99,21 +68,38 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): self.logger.debug(proc.stderr.decode()) proc.check_returncode() + def check_mergerepo_c(self): + cmd = ['/usr/bin/mergerepo_c', '--version'] + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out, _ = proc.communicate() + status = proc.wait() + if status != 0: + self.logger.warning("Unable to detect mergerepo_c version") + return False + except Exception: + self.logger.warning("Unable to detect mergerepo_c version") + return False + return True + def merge_arch_repo(self, arch, repodir, srcdir): archdir = joinpath(repodir, arch) - self.logger.debug(f'Creating {arch} repo under {archdir}') + self.logger.debug('Creating %s repo under %s' % (arch, archdir)) koji.ensuredir(archdir) - cmd = [ - '/usr/bin/mergerepo_c', - '--koji', - '--database', - f'--outputdir={archdir}', - f'--archlist={arch},noarch,src', - '--compress-type=gz', - ] + if self.check_mergerepo_c: + cmd = [ + '/usr/bin/mergerepo_c', + '--koji', + '--database', + '--outputdir=%s' % archdir, + '--archlist=%s,noarch,src' % arch, + '--compress-type=gz', + ] + else: + raise koji.GenericError("mergerepo_c is not installed") for srcrepo in os.listdir(srcdir): - cmd.append(f'--repo={srcrepo}') - self.logger.debug(f'Running: {" ".join(cmd)}') + cmd.append('--repo=%s' % srcrepo) + self.logger.debug('Running: %s' " ".join(cmd)) proc = subprocess.run(cmd, cwd=srcdir, capture_output=True) self.logger.debug(proc.stdout.decode()) if proc.stderr: @@ -133,65 +119,44 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): def create_repos(self, taskinfo): builds = self.session.listBuilds(taskID=taskinfo['id']) build = None - task_id = str(self.id) if builds: - build = builds[0] - rpms = self.session.listRPMs(buildID=build['build_id']) - repodir = joinpath(self.workdir, 'taskrepos', task_id, task_id, - build['name'], build['version'], build['release']) - nvr = f"{build['name']}.{build['version']}.{build['release']}" + info = builds[0] + rpms = self.session.listRPMs(buildID=info['build_id']) + repotype = 'official' else: - children = self.session.getTaskChildren(taskinfo['id'], request=True) - srpminfo = None + info = None rpms = [] - for child in children: - if child['method'] != 'buildArch': - continue - results = self.session.getTaskResult(child['id']) - if results.get('srpms'): - rpms.append(results['srpms'][0]) - srpmname = os.path.basename(results['srpms'][0]) - srpminfo = koji.parse_NVRA(srpmname) - rpms.extend(results['rpms']) - assert srpminfo, 'missing srpm' - repodir = joinpath(self.workdir, 'taskrepos', task_id, task_id, 'scratch', - srpminfo['name'], srpminfo['version'], srpminfo['release']) - nvr = f"{srpminfo['name']}.{srpminfo['version']}.{srpminfo['release']}" + results = self.session.getTaskResult(taskinfo['id']) + if results.get('srpms'): + rpms.append(results['srpms'][0]) + srpmname = os.path.basename(results['srpms'][0]) + info = koji.parse_NVRA(srpmname) + rpms.extend(results['rpms']) + if info is None: + raise koji.GenericError("SRPM is missing.") + repotype = 'scratch' + repodir = joinpath(self.workdir, info['name'], info['version'], info['release']) + repodir_nvr = joinpath(repotype, info['name'], info['version'], info['release']) + nvr = '%s-%s-%s' % (info['name'], info['version'], info['release']) + koji.ensuredir(repodir) - dirfd = os.open(repodir, os.O_RDONLY) - fcntl.flock(dirfd, fcntl.LOCK_EX) - self.rm_contents(repodir) - content_dir = joinpath(repodir, '_content') - self.mirror_rpms(rpms, content_dir, build=build) pi = koji.PathInfo(self.options.topurl) - for subdir in os.listdir(content_dir): + for arch in self.build_arches(rpms): if build: - baseurl = pi.build(build) + '/' + subdir + baseurl = pi.build(build) + '/' + arch else: - baseurl = pi.task(int(subdir)) - self.gen_repodata(joinpath(content_dir, subdir), baseurl) + baseurl = pi.task(int(taskinfo['id'])) + self.gen_repodata(joinpath(repodir, arch), baseurl) for arch in self.build_arches(rpms): - self.merge_arch_repo(arch, repodir, content_dir) - rmtree(content_dir) - return repodir, baseurl, nvr - - def link_task_repos(self, task_id, repodir): - """Symlink repodir to $basedir/tasks/{task_id}""" - tasksdir = joinpath(self.workdir, 'tasks') - if not os.path.exists(tasksdir): - koji.ensuredir(tasksdir) - dstdir = joinpath(tasksdir, str(task_id)) - if os.path.islink(dstdir) or os.path.isfile(dstdir): - os.remove(dstdir) - elif os.path.isdir(dstdir): - rmtree(dstdir) - os.symlink(os.path.relpath(repodir, tasksdir), dstdir) - - def write_repo_file(self, task, repodir): - nvr = '-'.join(repodir.split('/')[-3:]) + self.merge_arch_repo(arch, repodir, repodir) + return repodir, repodir_nvr, nvr, repotype + + def write_repo_file(self, task, repodir, nvr, repotype): if len(task['request']) > 2: - scratch = task['request'][2].get('scratch', False) + for tr in task['request']: + if isinstance(tr, dict): + scratch = tr.get('scratch', False) else: scratch = False subtasks = [] @@ -199,38 +164,42 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): subtasks = [subtask for subtask in task.getChildren(task['id'], request=True)] repo_arches = [] for subtask in subtasks: - repo_arches.append(f'{self.options.topurl}/{subtask["label"]}') + repo_arches.append('%s/%s' % (self.options.topurl, subtask["label"])) + rpminfo = koji.parse_NVR(nvr) + baseurl = joinpath(self.options.topurl, 'repos-tasks', repotype, + rpminfo['name'], rpminfo['version'], rpminfo['release']) if 'noarch' in repo_arches: - repo_url = f'{self.options.topurl}/noarch/' + baseurl = '%s/noarch/' % baseurl else: - repo_url = f'{self.options.topurl}/$basearch/' + baseurl = '%s/%s/' % (baseurl, '$basearch') repo_file_name = nvr - repo_name = f'brew-task-repo-{nvr.replace('+', '_')}' + repo_name = 'brew-task-repo-%s' % nvr.replace('+', '_') build = 'build' if scratch: repo_file_name += '-scratch' repo_name += '-scratch' build = 'scratch build' repo_file_name += '.repo' - repo_file = f'{repodir}/{repo_file_name}' + repo_file = '%s/%s' % (repodir, repo_file_name) with koji._open_text_file(repo_file, 'w') as repo_fd: repo_fd.write( - f"""[{repo_name}] - name=Repo for Brew {build} of {nvr} - enabled=1 - gpgcheck=0 - baseurl={repo_url} - module_hotfixes=1 - """ + """[%s] +name=Repo for Brew %s of %s +enabled=1 +gpgcheck=0 +baseurl=%s +module_hotfixes=1 +""" % (repo_name, build, nvr, baseurl) ) - self.logger.debug(f'Wrote repo file to {repo_file}') - return repo_file_name + self.logger.debug('Wrote repo file to %s' % repo_file) + return baseurl def upload_repo(self, repodir): repo_files = [] - repo_paths = [] - uploadpath_basic = "taskrepos/%s/%s" % (self.id % 10000, self.id) + repos = [] + repopath_part = '/'.join(repodir.split('/')[-4:]) + uploadpath_basic = joinpath("repos-tasks", repopath_part) for dirpath, dirs, files in os.walk(repodir): relrepodir = os.path.relpath(dirpath, repodir) for filename in files: @@ -247,11 +216,13 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): uploadpath = uploadpath_basic fn = relpath self.session.uploadWrapper(localpath, uploadpath, fn) - repo_paths.append(joinpath(uploadpath, filename)) + repos.append([uploadpath, fn]) repo_files.append(relpath) - return uploadpath_basic, repo_files, repo_paths + return repo_files, repos, repopath_part def handler(self, task_id_child, task_link=False): + read_config() + data = {} if not isinstance(task_id_child, int): task_id_child = int(task_id_child) taskinfo = self.session.getTaskInfo(task_id_child, request=True) @@ -260,96 +231,56 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): } self.session.host.assertPolicy('taskrepos', policy_data) if not taskinfo: - raise koji.BuildError(f'Invalid task ID: {task_id_child}') - if taskinfo['method'] != 'build': - raise koji.BuildError(f'{task_id_child} is not a build task') - build_subtask = next(task for task in self.session.getTaskChildren(task_id_child) - if task['method'] == 'buildArch') - if build_subtask['state'] != 2: - raise koji.BuildError(f'task {build_subtask['id']} has not completed successfully') - repodir, baseurl, nvr = self.create_repos(taskinfo) - self.logger.debug(f'Repos for task {task_id_child} created under {repodir}') - if task_link: - self.link_task_repos(task_id_child, repodir) - self.write_repo_file(taskinfo, repodir) - uploadpath, repo_files, repo_paths = self.upload_repo(repodir) - owner_id = taskinfo['owner'] - owner = self.session.getUser(owner_id)['name'] - self.session.tasksRepoNotifications(self.id, repodir, baseurl, owner, repo_paths, nvr) + raise koji.BuildError('Invalid task ID: %s' % task_id_child) + if taskinfo['method'] != 'buildArch': + raise koji.BuildError('%s is not a build task' % task_id_child) + if taskinfo['state'] != 2: + raise koji.BuildError('task %s has not completed successfully' % taskinfo['id']) + repodir, repodir_nvr, data['nvr'], repotype = self.create_repos(taskinfo) + self.logger.debug('Repos for task %s created under %s' % (task_id_child, repodir)) + data['baseurl'] = self.write_repo_file(taskinfo, repodir, data['nvr'], repotype) + repo_files, repos, repopath_part = self.upload_repo(repodir) + data['owner'] = self.session.getUser(taskinfo['owner'])['name'] + data['repo_urls'], uploadpath = self.session.taskRepoDone( + self.id, repos, repodir_nvr, self.options.topurl, task_link) + data['repodir'] = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr) + self.session.host.tasksRepoNotifications(self.id, data) return [uploadpath, repo_files] class tasksRepoNotifications(BaseTaskHandler): Methods = ['tasksRepoNotifications'] - _taskWeight = 0.1 + INI_FILE = '/etc/kojid/plugins/taskrepos.ini' + ini_config = '' - def __init__(self, *args, **kwargs): - self._read_config() - return super(tasksRepoNotifications, self).__init__(*args, **kwargs) - - def _read_config(self): - cp = koji.read_config_files(CONFIG_FILE) - self.config = { - 'REPO_LIFETIME': 14, # days - 'TICKETLINK': '', - 'SOURCECODE': '', + def read_ini_file(self): + cp_ini = koji.read_config_files(self.INI_FILE) + self.ini_config = { + 'email_template': '', } # expire repos in days - if cp.has_option('taskrepos', 'expire_repos'): - self.config['REPO_LIFETIME'] = cp.get('taskrepos', 'expire_repos') - - if cp.has_option('taskrepos', 'ticketlink'): - self.config['TICKETLINK'] = cp.get('taskrepos', 'ticketlink') - - if cp.has_option('taskrepos', 'sourcecodelink'): - self.config['SOURCECODE'] = cp.get('taskrepos', 'sourcecodelink') - - # XXX externalize these templates somewhere - subject_templ = 'repo for %(build)s of %(nvr)s is available' - message_templ = \ - """From: "Brew Task Repos System" %(from_addr)s\r -Subject: %(subject)s\r -To: %(recipient)s\r -A yum repository for the %(build)s of %(nvr)s (task %(task_id)s) is available at:\r -\r -%(repodir)s/\r -\r -You can install the rpms locally by putting this .repo file in your /etc/yum.repos.d/ directory:\r -\r -%(repofile)s\r -\r -It is necessary to have internal Red Hat CA certificates installed on the system, if -you want to use yum repository. You can install required certificates from:\r -\r -* http://hdn.corp.redhat.com/rhel7-csb-stage/repoview/redhat-internal-cert-install.html (rpm)\r -* https://certs.corp.redhat.com/ (cert files)\r -\r -The full list of repos is:\r -%(repo_urls)s\r -\r -The repository will be available for the next %(repolifetime)s days. -Scratch build output will be deleted earlier, based on the Brew scratch build retention policy.\r -\r -If you found a bug or you wish to stop these emails, please create a ticket:\r -%(ticketlink)s\r -\r -Source code at %(sourcecode)s\r -""" - - def handler(self, recipient, repo_urls, task_id, repodir, repofile, nvr): - if len(recipient) == 0: - self.logger.debug(f'task {self.id}: no recipients, not sending notifications') - return - from_addr = self.options.from_addr - ticketlink = self.config['TICKETLINK'] - sourcecode = self.config['SOURCECODE'] - repolifetime = self.config['REPO_LIFETIME'] - build = nvr.split('.')[0] - repo_urls = '\n'.join(repo_urls) - subject = self.subject_templ % locals() - message = self.message_templ % locals() + if cp_ini.has_option('EMAIL TEMPLATE', 'email_template'): + self.ini_config['email_template'] = cp_ini.get('EMAIL TEMPLATE', 'email_template') + + def handler(self, recipient, data): + read_config() + self.read_ini_file() + data['from_addr'] = self.options.from_addr + repo_urls = data['repo_urls'].copy() + data['repo_urls'] = '' + for rf in repo_urls: + if '.repo' in rf: + data['repofile'] = rf + else: + data['repo_urls'] += '%s\n' % rf + data['ticketlink'] = config['ticketlink'] + data['sourcecode'] = config['sourcecode'] + data['repolifetime'] = config['repo_lifetime'] + data['build'] = '-'.join(data['nvr'].split('-')[:-2]) + data['recipient'] = recipient + message = config['email_template'] % data # ensure message is in UTF-8 message = koji.fixEncoding(message) @@ -360,7 +291,7 @@ Source code at %(sourcecode)s\r if self.options.smtp_user is not None and self.options.smtp_pass is not None: server.login(self.options.smtp_user, self.options.smtp_pass) # server.set_debuglevel(True) - server.sendmail(from_addr, recipient, message) + server.sendmail(data['from_addr'], recipient, message) server.quit() - return f'sent notification of taskrepo {self.id} to: {recipient}' + return 'sent notification of taskrepo %s to: %s' % (self.id, recipient) diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py index 2832d46..a91cb16 100644 --- a/plugins/hub/taskrepos_hub.py +++ b/plugins/hub/taskrepos_hub.py @@ -4,38 +4,71 @@ import kojihub +import logging +import os +import shutil +import koji from koji.context import context -from koji.plugin import export, callback -from koji.util import joinpath +from koji.plugin import export, export_in, callback +from koji.util import joinpath, safer_move, rmtree -__all__ = ('taskrepos',) +__all__ = ('taskrepos', 'tasksRepoNotifications', 'taskRepoDone') -@callback('postBuildStateChange') +logger = logging.getLogger('koji.hub') + + +@export +def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): + repo_urls = [] + nvr_dir = '/'.join(repodir_nvr.split('/')[1:]) + '/' + for rp, fn in repos: + src = joinpath(koji.pathinfo.work(), rp, fn).replace('/.', '/') + repostasks_dir = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr, + rp.split(nvr_dir)[1]).replace('/.', '/') + repostasks_file = joinpath(repostasks_dir, fn).replace('/.', '/') + koji.ensuredir(repostasks_dir) + if not os.path.exists(src): + raise koji.GenericError("uploaded file missing: %s" % src) + safer_move(src, repostasks_file) + repo_urls.append(joinpath(topurl, 'repos-tasks', repodir_nvr, + rp.split(nvr_dir)[1], fn).replace('/.', '/')) + if task_link: + repodir = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr) + task_dir_with_id = joinpath(koji.pathinfo.topdir, 'tasks', str(task_id % 10000)) + koji.ensuredir(task_dir_with_id) + dstdir = joinpath(task_dir_with_id, str(task_id)) + if os.path.islink(dstdir) or os.path.isfile(dstdir): + os.remove(dstdir) + elif os.path.isdir(dstdir): + rmtree(dstdir) + os.symlink(repodir, dstdir) + shutil.rmtree(joinpath( + koji.pathinfo.work(), '/'.join(rp.split(str(task_id))[:1]), str(task_id))) + return repo_urls, repostasks_dir + + +@callback('postTaskStateChange') def taskrepos(cbtype, *args, **kws): - task_id = kws['info']['task_id'] + if kws['attribute'] != 'state': + return + task_id = kws['info']['id'] task = kojihub.Task(task_id) - subtasks_method = [subtask['method'] for subtask in task.getChildren(task_id)] - if 'buildArch' not in subtasks_method: + taskinfo = task.getInfo() + if taskinfo['method'] != 'buildArch' or taskinfo['state'] != koji.TASK_STATES['CLOSED']: return - task_link = False + task_link = True kojihub.make_task('taskrepos', [task_id, task_link]) - return -@export -def tasksRepoNotifications(task_id, repodir, baseurl, owner, repo_paths, nvr): +@export_in('host') +def tasksRepoNotifications(task_id, data): if context.opts.get('DisableNotifications'): return email_domain = context.opts['EmailDomain'] - email = f'{owner}@{email_domain}' - repo_urls = [] - for rf in repo_paths: - if '.repo' in rf: - repofile = joinpath(baseurl, rf) - else: - repo_urls.append(joinpath(baseurl, rf)) - kojihub.make_task('tasksRepoNotifications', - [email, repo_urls, task_id, repodir, repofile, nvr]) + email = f'{data['owner']}@{email_domain}' + + data['task_id'] = task_id + kojihub.make_task('tasksRepoNotifications', [email, data]) From f475ec216fe26407b2e7f5d69139a027a218dc50 Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 3/11] Fix review --- diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index 6fc99ab..b49452a 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -33,7 +33,7 @@ def read_config(): # expire repos in days if cp.has_option('taskrepos', 'expire_repos'): - config['repo_lifetime'] = cp.get('taskrepos', 'expire_repos') + config['repo_lifetime'] = cp.getint('taskrepos', 'expire_repos') if cp.has_option('taskrepos', 'ticketlink'): config['ticketlink'] = cp.get('taskrepos', 'ticketlink') @@ -68,35 +68,18 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): self.logger.debug(proc.stderr.decode()) proc.check_returncode() - def check_mergerepo_c(self): - cmd = ['/usr/bin/mergerepo_c', '--version'] - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) - out, _ = proc.communicate() - status = proc.wait() - if status != 0: - self.logger.warning("Unable to detect mergerepo_c version") - return False - except Exception: - self.logger.warning("Unable to detect mergerepo_c version") - return False - return True - def merge_arch_repo(self, arch, repodir, srcdir): archdir = joinpath(repodir, arch) self.logger.debug('Creating %s repo under %s' % (arch, archdir)) koji.ensuredir(archdir) - if self.check_mergerepo_c: - cmd = [ - '/usr/bin/mergerepo_c', - '--koji', - '--database', - '--outputdir=%s' % archdir, - '--archlist=%s,noarch,src' % arch, - '--compress-type=gz', - ] - else: - raise koji.GenericError("mergerepo_c is not installed") + cmd = [ + '/usr/bin/mergerepo_c', + '--koji', + '--database', + '--outputdir=%s' % archdir, + '--archlist=%s,noarch,src' % arch, + '--compress-type=gz', + ] for srcrepo in os.listdir(srcdir): cmd.append('--repo=%s' % srcrepo) self.logger.debug('Running: %s' " ".join(cmd)) diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py index a91cb16..48ae2e1 100644 --- a/plugins/hub/taskrepos_hub.py +++ b/plugins/hub/taskrepos_hub.py @@ -68,7 +68,7 @@ def tasksRepoNotifications(task_id, data): if context.opts.get('DisableNotifications'): return email_domain = context.opts['EmailDomain'] - email = f'{data['owner']}@{email_domain}' + email = f"{data['owner']}@{email_domain}" data['task_id'] = task_id kojihub.make_task('tasksRepoNotifications', [email, data]) From 25911846f880bac3be8551a2213c754c3623f588 Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 4/11] Fix review --- diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index b49452a..c4bfac1 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -6,6 +6,7 @@ import os import six import smtplib import subprocess +import time import koji import koji.tasks @@ -50,15 +51,16 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): Methods = ['taskrepos'] _taskWeight = 2.0 - def gen_repodata(self, rpmdir, baseurl): + def gen_repodata(self, rpmdir): koji.ensuredir(rpmdir) cmd = [ '/usr/bin/createrepo_c', '--database', - '--baseurl=%s' % baseurl, + '--outputdir=%s' % rpmdir, '--checksum=sha256', '--general-compress-type=gz', - rpmdir, + '--pkglist=pkglist', + self.options.topdir, ] self.logger.debug('Running: %s' % " ".join(cmd)) @@ -99,6 +101,43 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): arches.add(rpm['arch']) return arches + def create_pkg_list(self, rpms, repodir, build=None): + remote_pi = koji.PathInfo(topdir=self.options.topdir) + dest_dirs = [] + pkglist = [] + for rpm in rpms: + if build: + # rpm is a dict of rpminfo + url = remote_pi.build(build) + '/' + remote_pi.rpm(rpm) + dest = joinpath(repodir, rpm['arch']) + else: + # rpm is a relative path under the work/ directory + taskdir, rpmname = rpm.split('/')[-2:] + url = joinpath(remote_pi.work(), rpm).replace(self.options.topdir, '', 1) + rpminfo = self.session.getRPM(rpmname) + count = 0 + while rpminfo is None and count != 20: + time.sleep(2) + rpminfo = self.session.getRPM(rpmname) + count += 1 + if rpminfo['arch'] != 'src': + dest = joinpath(repodir, rpminfo['arch']) + else: + continue + if dest not in dest_dirs: + dest_dirs.append(dest) + pkglist.append(url) + for dest_dir in dest_dirs: + os.makedirs(dest_dir, exist_ok=True) + os.symlink(koji.pathinfo.topdir, joinpath(dest_dir, 'toplink')) + with open(joinpath(dest_dir, 'pkglist'), 'w') as dd: + for pkg in pkglist: + dd.write(pkg) + with open(joinpath(dest_dir, 'pkglist'), 'r') as dd: + for line in dd: + self.logger.info('PkgLINE %s' % line) + return dest_dirs + def create_repos(self, taskinfo): builds = self.session.listBuilds(taskID=taskinfo['id']) build = None @@ -124,13 +163,9 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): koji.ensuredir(repodir) - pi = koji.PathInfo(self.options.topurl) + self.create_pkg_list(rpms, repodir, build=build) for arch in self.build_arches(rpms): - if build: - baseurl = pi.build(build) + '/' + arch - else: - baseurl = pi.task(int(taskinfo['id'])) - self.gen_repodata(joinpath(repodir, arch), baseurl) + self.gen_repodata(joinpath(repodir, arch)) for arch in self.build_arches(rpms): self.merge_arch_repo(arch, repodir, repodir) return repodir, repodir_nvr, nvr, repotype From 2110fcf96c3b02bd638c1873e4e8edbeb7960362 Mon Sep 17 00:00:00 2001 From: Jana Librova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 5/11] Fix review --- diff --git a/plugins/builder/taskrepos.conf b/plugins/builder/taskrepos.conf index 7769b2f..1fa3e61 100644 --- a/plugins/builder/taskrepos.conf +++ b/plugins/builder/taskrepos.conf @@ -7,6 +7,7 @@ email_template : From: %%(from_addr)s To: %%(recipient)s A yum repository for the %%(build)s of %%(nvr)s (task %%(task_id)s) is available at: %%(repodir)s/ + %%(old_repodir_msg)s You can install the rpms locally by putting this .repo file in your /etc/yum.repos.d/ directory: %%(repofile)s diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index c4bfac1..1361062 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -6,7 +6,6 @@ import os import six import smtplib import subprocess -import time import koji import koji.tasks @@ -102,28 +101,13 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): return arches def create_pkg_list(self, rpms, repodir, build=None): - remote_pi = koji.PathInfo(topdir=self.options.topdir) + remote_pi = koji.PathInfo(topdir='') dest_dirs = [] pkglist = [] for rpm in rpms: - if build: - # rpm is a dict of rpminfo - url = remote_pi.build(build) + '/' + remote_pi.rpm(rpm) - dest = joinpath(repodir, rpm['arch']) - else: - # rpm is a relative path under the work/ directory - taskdir, rpmname = rpm.split('/')[-2:] - url = joinpath(remote_pi.work(), rpm).replace(self.options.topdir, '', 1) - rpminfo = self.session.getRPM(rpmname) - count = 0 - while rpminfo is None and count != 20: - time.sleep(2) - rpminfo = self.session.getRPM(rpmname) - count += 1 - if rpminfo['arch'] != 'src': - dest = joinpath(repodir, rpminfo['arch']) - else: - continue + url = joinpath(remote_pi.work(), rpm) + arch = rpm.split('.')[-2] + dest = joinpath(repodir, arch) if dest not in dest_dirs: dest_dirs.append(dest) pkglist.append(url) @@ -132,31 +116,31 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): os.symlink(koji.pathinfo.topdir, joinpath(dest_dir, 'toplink')) with open(joinpath(dest_dir, 'pkglist'), 'w') as dd: for pkg in pkglist: - dd.write(pkg) + dd.write(f'{pkg}\n') with open(joinpath(dest_dir, 'pkglist'), 'r') as dd: for line in dd: self.logger.info('PkgLINE %s' % line) return dest_dirs def create_repos(self, taskinfo): - builds = self.session.listBuilds(taskID=taskinfo['id']) + builds = self.session.listBuilds(taskID=taskinfo['parent']) build = None if builds: - info = builds[0] - rpms = self.session.listRPMs(buildID=info['build_id']) + build = builds[0] repotype = 'official' else: - info = None - rpms = [] - results = self.session.getTaskResult(taskinfo['id']) - if results.get('srpms'): - rpms.append(results['srpms'][0]) - srpmname = os.path.basename(results['srpms'][0]) - info = koji.parse_NVRA(srpmname) - rpms.extend(results['rpms']) - if info is None: - raise koji.GenericError("SRPM is missing.") repotype = 'scratch' + info = None + rpms = [] + results = self.session.getTaskResult(taskinfo['id']) + if results.get('srpms'): + srpm = results['srpms'][0] + srpmname = os.path.basename(results['srpms'][0]) + info = koji.parse_NVRA(srpmname) + if info is None: + raise koji.GenericError("SRPM is missing.") + if results.get('rpms'): + rpms.extend(results['rpms']) repodir = joinpath(self.workdir, info['name'], info['version'], info['release']) repodir_nvr = joinpath(repotype, info['name'], info['version'], info['release']) nvr = '%s-%s-%s' % (info['name'], info['version'], info['release']) @@ -164,6 +148,7 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): koji.ensuredir(repodir) self.create_pkg_list(rpms, repodir, build=build) + rpms.append(srpm) for arch in self.build_arches(rpms): self.gen_repodata(joinpath(repodir, arch)) for arch in self.build_arches(rpms): @@ -171,22 +156,17 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): return repodir, repodir_nvr, nvr, repotype def write_repo_file(self, task, repodir, nvr, repotype): - if len(task['request']) > 2: - for tr in task['request']: + parent_taskinfo = self.session.getTaskInfo(task['parent'], request=True) + if len(parent_taskinfo['request']) > 2: + for tr in parent_taskinfo['request']: if isinstance(tr, dict): scratch = tr.get('scratch', False) else: scratch = False - subtasks = [] - if task['method'] == 'createrepo': - subtasks = [subtask for subtask in task.getChildren(task['id'], request=True)] - repo_arches = [] - for subtask in subtasks: - repo_arches.append('%s/%s' % (self.options.topurl, subtask["label"])) rpminfo = koji.parse_NVR(nvr) baseurl = joinpath(self.options.topurl, 'repos-tasks', repotype, rpminfo['name'], rpminfo['version'], rpminfo['release']) - if 'noarch' in repo_arches: + if 'noarch' == task["label"]: baseurl = '%s/noarch/' % baseurl else: baseurl = '%s/%s/' % (baseurl, '$basearch') @@ -259,7 +239,7 @@ module_hotfixes=1 data['baseurl'] = self.write_repo_file(taskinfo, repodir, data['nvr'], repotype) repo_files, repos, repopath_part = self.upload_repo(repodir) data['owner'] = self.session.getUser(taskinfo['owner'])['name'] - data['repo_urls'], uploadpath = self.session.taskRepoDone( + data['repo_urls'], uploadpath, data['old_repodir'] = self.session.taskRepoDone( self.id, repos, repodir_nvr, self.options.topurl, task_link) data['repodir'] = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr) self.session.host.tasksRepoNotifications(self.id, data) @@ -293,6 +273,13 @@ class tasksRepoNotifications(BaseTaskHandler): data['repofile'] = rf else: data['repo_urls'] += '%s\n' % rf + + if data['old_repodir']: + data['old_repodir_msg'] = (f"\nThis is latest version of repo. When you would like" + f" to use original version of repo, please use " + f"{data['old_repodir']}\n") + else: + data['old_repodir_msg'] = '' data['ticketlink'] = config['ticketlink'] data['sourcecode'] = config['sourcecode'] data['repolifetime'] = config['repo_lifetime'] diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py index 48ae2e1..33c9a72 100644 --- a/plugins/hub/taskrepos_hub.py +++ b/plugins/hub/taskrepos_hub.py @@ -19,34 +19,57 @@ __all__ = ('taskrepos', 'tasksRepoNotifications', 'taskRepoDone') logger = logging.getLogger('koji.hub') +def recursive_walk(dirpath, repo_urls, topurl, task_link): + for root, dir, files in os.walk(dirpath): + if dir: + for subfolder in dir: + recursive_walk(subfolder, repo_urls, topurl, task_link) + if task_link: + folder_name_part = root[root.find('repos-tasks'):] + else: + folder_name_part = root[root.find('tasks'):] + for filename in files: + repo_urls.append(joinpath(topurl, folder_name_part, filename)) + return repo_urls + + @export def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): repo_urls = [] + old_repodir = None nvr_dir = '/'.join(repodir_nvr.split('/')[1:]) + '/' for rp, fn in repos: src = joinpath(koji.pathinfo.work(), rp, fn).replace('/.', '/') - repostasks_dir = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr, - rp.split(nvr_dir)[1]).replace('/.', '/') - repostasks_file = joinpath(repostasks_dir, fn).replace('/.', '/') - koji.ensuredir(repostasks_dir) + task_dir = joinpath(koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), + str(task_id), rp.split(nvr_dir)[1]).replace('/.', '/') + taskfile = joinpath(task_dir, fn).replace('/.', '/') + koji.ensuredir(task_dir) if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) - safer_move(src, repostasks_file) - repo_urls.append(joinpath(topurl, 'repos-tasks', repodir_nvr, - rp.split(nvr_dir)[1], fn).replace('/.', '/')) + safer_move(src, taskfile) + if task_link: - repodir = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr) - task_dir_with_id = joinpath(koji.pathinfo.topdir, 'tasks', str(task_id % 10000)) - koji.ensuredir(task_dir_with_id) - dstdir = joinpath(task_dir_with_id, str(task_id)) - if os.path.islink(dstdir) or os.path.isfile(dstdir): - os.remove(dstdir) - elif os.path.isdir(dstdir): - rmtree(dstdir) - os.symlink(repodir, dstdir) + task_dir_with_id = joinpath( + koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), str(task_id)) + reponvr_dir = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr) + koji.ensuredir(reponvr_dir) + if os.path.islink(reponvr_dir) or os.path.isfile(reponvr_dir): + old_repodir = os.readlink(reponvr_dir) + os.remove(reponvr_dir) + elif os.path.isdir(reponvr_dir): + rmtree(reponvr_dir) + os.symlink(task_dir_with_id, reponvr_dir) + + if task_link: + files_dir = reponvr_dir + else: + files_dir = joinpath( + koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), str(task_id)) + repo_urls = recursive_walk(files_dir, repo_urls, topurl, task_link) + shutil.rmtree(joinpath( koji.pathinfo.work(), '/'.join(rp.split(str(task_id))[:1]), str(task_id))) - return repo_urls, repostasks_dir + return repo_urls, files_dir, old_repodir @callback('postTaskStateChange') From c3b4ceb78168184d210fb1d11c9565f263c449e0 Mon Sep 17 00:00:00 2001 From: Jana Librova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 6/11] Fix code review --- diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index 1361062..fd80c13 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -100,7 +100,7 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): arches.add(rpm['arch']) return arches - def create_pkg_list(self, rpms, repodir, build=None): + def create_pkg_list(self, rpms, repodir): remote_pi = koji.PathInfo(topdir='') dest_dirs = [] pkglist = [] @@ -117,16 +117,12 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): with open(joinpath(dest_dir, 'pkglist'), 'w') as dd: for pkg in pkglist: dd.write(f'{pkg}\n') - with open(joinpath(dest_dir, 'pkglist'), 'r') as dd: - for line in dd: - self.logger.info('PkgLINE %s' % line) + self.logger.info('PkgLINE %s' % pkg) return dest_dirs def create_repos(self, taskinfo): builds = self.session.listBuilds(taskID=taskinfo['parent']) - build = None if builds: - build = builds[0] repotype = 'official' else: repotype = 'scratch' @@ -147,15 +143,15 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): koji.ensuredir(repodir) - self.create_pkg_list(rpms, repodir, build=build) + self.create_pkg_list(rpms, repodir) rpms.append(srpm) for arch in self.build_arches(rpms): self.gen_repodata(joinpath(repodir, arch)) for arch in self.build_arches(rpms): self.merge_arch_repo(arch, repodir, repodir) - return repodir, repodir_nvr, nvr, repotype + return repodir, repodir_nvr, nvr - def write_repo_file(self, task, repodir, nvr, repotype): + def write_repo_file(self, task, repodir, nvr): parent_taskinfo = self.session.getTaskInfo(task['parent'], request=True) if len(parent_taskinfo['request']) > 2: for tr in parent_taskinfo['request']: @@ -163,9 +159,8 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): scratch = tr.get('scratch', False) else: scratch = False - rpminfo = koji.parse_NVR(nvr) - baseurl = joinpath(self.options.topurl, 'repos-tasks', repotype, - rpminfo['name'], rpminfo['version'], rpminfo['release']) + baseurl = joinpath(self.options.topurl, 'repos-tasks', 'tasks', str(self.id % 10000), + str(self.id)) if 'noarch' == task["label"]: baseurl = '%s/noarch/' % baseurl else: @@ -197,7 +192,6 @@ module_hotfixes=1 repo_files = [] repos = [] repopath_part = '/'.join(repodir.split('/')[-4:]) - uploadpath_basic = joinpath("repos-tasks", repopath_part) for dirpath, dirs, files in os.walk(repodir): relrepodir = os.path.relpath(dirpath, repodir) for filename in files: @@ -208,10 +202,10 @@ module_hotfixes=1 localpath = '%s/%s' % (repodir, relpath) reldir = os.path.dirname(relpath) if reldir: - uploadpath = "%s/%s" % (uploadpath_basic, reldir) + uploadpath = "%s/%s" % (repopath_part, reldir) fn = os.path.basename(relpath) else: - uploadpath = uploadpath_basic + uploadpath = repopath_part fn = relpath self.session.uploadWrapper(localpath, uploadpath, fn) repos.append([uploadpath, fn]) @@ -234,9 +228,9 @@ module_hotfixes=1 raise koji.BuildError('%s is not a build task' % task_id_child) if taskinfo['state'] != 2: raise koji.BuildError('task %s has not completed successfully' % taskinfo['id']) - repodir, repodir_nvr, data['nvr'], repotype = self.create_repos(taskinfo) + repodir, repodir_nvr, data['nvr'] = self.create_repos(taskinfo) self.logger.debug('Repos for task %s created under %s' % (task_id_child, repodir)) - data['baseurl'] = self.write_repo_file(taskinfo, repodir, data['nvr'], repotype) + data['baseurl'] = self.write_repo_file(taskinfo, repodir, data['nvr']) repo_files, repos, repopath_part = self.upload_repo(repodir) data['owner'] = self.session.getUser(taskinfo['owner'])['name'] data['repo_urls'], uploadpath, data['old_repodir'] = self.session.taskRepoDone( diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py index 33c9a72..5191ef9 100644 --- a/plugins/hub/taskrepos_hub.py +++ b/plugins/hub/taskrepos_hub.py @@ -6,7 +6,6 @@ import kojihub import logging import os -import shutil import koji from koji.context import context @@ -67,7 +66,7 @@ def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), str(task_id)) repo_urls = recursive_walk(files_dir, repo_urls, topurl, task_link) - shutil.rmtree(joinpath( + koji.util.rmtree(joinpath( koji.pathinfo.work(), '/'.join(rp.split(str(task_id))[:1]), str(task_id))) return repo_urls, files_dir, old_repodir From dd0f809a0a8970032bd8de8df131acd2e167c691 Mon Sep 17 00:00:00 2001 From: Jana Librova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 7/11] Fix code review --- diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index fd80c13..c301452 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -59,7 +59,8 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): '--checksum=sha256', '--general-compress-type=gz', '--pkglist=pkglist', - self.options.topdir, + '--location-prefix=""', + '.', ] self.logger.debug('Running: %s' % " ".join(cmd)) @@ -106,6 +107,7 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): pkglist = [] for rpm in rpms: url = joinpath(remote_pi.work(), rpm) + url = f'toplink{url}' arch = rpm.split('.')[-2] dest = joinpath(repodir, arch) if dest not in dest_dirs: From 26db919ac1ddcc3c3c496800a18f1f18ed419319 Mon Sep 17 00:00:00 2001 From: Jana Librova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 8/11] Fix code review --- diff --git a/plugins/builder/taskrepos.conf b/plugins/builder/taskrepos.conf index 1fa3e61..6d7c98c 100644 --- a/plugins/builder/taskrepos.conf +++ b/plugins/builder/taskrepos.conf @@ -26,4 +26,4 @@ email_template : From: %%(from_addr)s If you found a bug or you wish to stop these emails, please create a ticket: %%(ticketlink)s - Source code at %%(sourcecode)s \ No newline at end of file + Source code at %%(sourcecodelink)s \ No newline at end of file diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index c301452..d425fc5 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -1,21 +1,17 @@ # kojid plugin -from __future__ import absolute_import - import os -import six import smtplib import subprocess +import six + import koji -import koji.tasks -from koji.tasks import ( - BaseTaskHandler -) +from koji.tasks import BaseTaskHandler from koji.util import joinpath -__all__ = ('TaskReposTask', 'tasksRepoNotifications') +__all__ = ('TaskReposTask', 'taskRepoNotifications') CONFIG_FILE = '/etc/kojid/plugins/taskrepos.conf' config = None @@ -27,7 +23,7 @@ def read_config(): config = { 'repo_lifetime': 14, # days 'ticketlink': '', - 'sourcecode': '', + 'sourcecodelink': '', 'email_template': '', } @@ -39,13 +35,13 @@ def read_config(): config['ticketlink'] = cp.get('taskrepos', 'ticketlink') if cp.has_option('taskrepos', 'sourcecodelink'): - config['sourcecode'] = cp.get('taskrepos', 'sourcecodelink') + config['sourcecodelink'] = cp.get('taskrepos', 'sourcecodelink') if cp.has_option('taskrepos', 'email_template'): config['email_template'] = cp.get('taskrepos', 'email_template') -class TaskReposTask(koji.tasks.BaseTaskHandler): +class TaskReposTask(BaseTaskHandler): Methods = ['taskrepos'] _taskWeight = 2.0 @@ -122,22 +118,30 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): self.logger.info('PkgLINE %s' % pkg) return dest_dirs - def create_repos(self, taskinfo): - builds = self.session.listBuilds(taskID=taskinfo['parent']) - if builds: - repotype = 'official' - else: - repotype = 'scratch' + def create_repos(self, tasksinfo): + if len(tasksinfo) != 0: + builds = self.session.listBuilds(taskID=tasksinfo[0]['parent']) + if builds: + repotype = 'official' + else: + repotype = 'scratch' info = None rpms = [] - results = self.session.getTaskResult(taskinfo['id']) - if results.get('srpms'): + results = { + 'rpms': [], + 'srpms': [], + } + for taskinfo in tasksinfo: + task_result = self.session.getTaskResult(taskinfo['id']) + results['rpms'].extend(task_result['rpms']) + results['srpms'].extend(task_result['srpms']) + if results['srpms'] != []: srpm = results['srpms'][0] srpmname = os.path.basename(results['srpms'][0]) info = koji.parse_NVRA(srpmname) if info is None: raise koji.GenericError("SRPM is missing.") - if results.get('rpms'): + if results['rpms'] != []: rpms.extend(results['rpms']) repodir = joinpath(self.workdir, info['name'], info['version'], info['release']) repodir_nvr = joinpath(repotype, info['name'], info['version'], info['release']) @@ -153,20 +157,21 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): self.merge_arch_repo(arch, repodir, repodir) return repodir, repodir_nvr, nvr - def write_repo_file(self, task, repodir, nvr): - parent_taskinfo = self.session.getTaskInfo(task['parent'], request=True) - if len(parent_taskinfo['request']) > 2: - for tr in parent_taskinfo['request']: + def write_repo_file(self, taskinfo, buildarch_subtasks, repodir, nvr): + if len(taskinfo['request']) > 2: + for tr in taskinfo['request']: if isinstance(tr, dict): scratch = tr.get('scratch', False) else: scratch = False baseurl = joinpath(self.options.topurl, 'repos-tasks', 'tasks', str(self.id % 10000), str(self.id)) - if 'noarch' == task["label"]: - baseurl = '%s/noarch/' % baseurl - else: - baseurl = '%s/%s/' % (baseurl, '$basearch') + baseurls = [] + for subtaskinfo in buildarch_subtasks: + if 'noarch' == subtaskinfo["label"]: + baseurls.append('%s/noarch/' % baseurl) + else: + baseurls.append('%s/%s/' % (baseurl, '$basearch')) repo_file_name = nvr repo_name = 'brew-task-repo-%s' % nvr.replace('+', '_') build = 'build' @@ -178,15 +183,16 @@ class TaskReposTask(koji.tasks.BaseTaskHandler): repo_file = '%s/%s' % (repodir, repo_file_name) with koji._open_text_file(repo_file, 'w') as repo_fd: - repo_fd.write( - """[%s] + for baseurl in baseurls: + repo_fd.write( + """[%s] name=Repo for Brew %s of %s enabled=1 gpgcheck=0 baseurl=%s module_hotfixes=1 """ % (repo_name, build, nvr, baseurl) - ) + ) self.logger.debug('Wrote repo file to %s' % repo_file) return baseurl @@ -214,53 +220,42 @@ module_hotfixes=1 repo_files.append(relpath) return repo_files, repos, repopath_part - def handler(self, task_id_child, task_link=False): + def handler(self, task_id, task_link=False): read_config() data = {} - if not isinstance(task_id_child, int): - task_id_child = int(task_id_child) - taskinfo = self.session.getTaskInfo(task_id_child, request=True) + if not isinstance(task_id, int): + task_id = int(task_id) + taskinfo = self.session.getTaskInfo(task_id, request=True) policy_data = { 'user_id': taskinfo['owner'], } self.session.host.assertPolicy('taskrepos', policy_data) if not taskinfo: - raise koji.BuildError('Invalid task ID: %s' % task_id_child) - if taskinfo['method'] != 'buildArch': - raise koji.BuildError('%s is not a build task' % task_id_child) + raise koji.BuildError('Invalid task ID: %s' % task_id) + if taskinfo['method'] != 'build': + raise koji.BuildError('%s is not a build task' % task_id) if taskinfo['state'] != 2: raise koji.BuildError('task %s has not completed successfully' % taskinfo['id']) - repodir, repodir_nvr, data['nvr'] = self.create_repos(taskinfo) - self.logger.debug('Repos for task %s created under %s' % (task_id_child, repodir)) - data['baseurl'] = self.write_repo_file(taskinfo, repodir, data['nvr']) + buildarch_subtasks = [subtask for subtask in self.session.getTaskChildren(task_id) + if subtask['method'] == 'buildArch'] + repodir, repodir_nvr, data['nvr'] = self.create_repos(buildarch_subtasks) + self.logger.debug('Repos for task %s created under %s' % (task_id, repodir)) + data['baseurl'] = self.write_repo_file(taskinfo, buildarch_subtasks, repodir, data['nvr']) repo_files, repos, repopath_part = self.upload_repo(repodir) data['owner'] = self.session.getUser(taskinfo['owner'])['name'] - data['repo_urls'], uploadpath, data['old_repodir'] = self.session.taskRepoDone( + data['repo_urls'], uploadpath, data['old_repodir'] = self.session.host.taskRepoDone( self.id, repos, repodir_nvr, self.options.topurl, task_link) data['repodir'] = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr) - self.session.host.tasksRepoNotifications(self.id, data) + self.session.host.taskRepoNotifications(self.id, data) return [uploadpath, repo_files] -class tasksRepoNotifications(BaseTaskHandler): - Methods = ['tasksRepoNotifications'] +class taskRepoNotifications(BaseTaskHandler): + Methods = ['taskRepoNotifications'] _taskWeight = 0.1 - INI_FILE = '/etc/kojid/plugins/taskrepos.ini' - ini_config = '' - - def read_ini_file(self): - cp_ini = koji.read_config_files(self.INI_FILE) - self.ini_config = { - 'email_template': '', - } - - # expire repos in days - if cp_ini.has_option('EMAIL TEMPLATE', 'email_template'): - self.ini_config['email_template'] = cp_ini.get('EMAIL TEMPLATE', 'email_template') def handler(self, recipient, data): read_config() - self.read_ini_file() data['from_addr'] = self.options.from_addr repo_urls = data['repo_urls'].copy() data['repo_urls'] = '' @@ -277,7 +272,7 @@ class tasksRepoNotifications(BaseTaskHandler): else: data['old_repodir_msg'] = '' data['ticketlink'] = config['ticketlink'] - data['sourcecode'] = config['sourcecode'] + data['sourcecodelink'] = config['sourcecodelink'] data['repolifetime'] = config['repo_lifetime'] data['build'] = '-'.join(data['nvr'].split('-')[:-2]) data['recipient'] = recipient @@ -292,7 +287,7 @@ class tasksRepoNotifications(BaseTaskHandler): if self.options.smtp_user is not None and self.options.smtp_pass is not None: server.login(self.options.smtp_user, self.options.smtp_pass) # server.set_debuglevel(True) - server.sendmail(data['from_addr'], recipient, message) + server.sendmail(data['from_addr'], recipient, message.decode()) server.quit() return 'sent notification of taskrepo %s to: %s' % (self.id, recipient) diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py index 5191ef9..6a4ded0 100644 --- a/plugins/hub/taskrepos_hub.py +++ b/plugins/hub/taskrepos_hub.py @@ -3,16 +3,16 @@ # plugin has a config file. This hub plugin has no config file. -import kojihub import logging import os import koji +import kojihub from koji.context import context -from koji.plugin import export, export_in, callback +from koji.plugin import export_in, callback from koji.util import joinpath, safer_move, rmtree -__all__ = ('taskrepos', 'tasksRepoNotifications', 'taskRepoDone') +__all__ = ('taskrepos', 'taskRepoNotifications', 'taskRepoDone') logger = logging.getLogger('koji.hub') @@ -32,7 +32,7 @@ def recursive_walk(dirpath, repo_urls, topurl, task_link): return repo_urls -@export +@export_in('host') def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): repo_urls = [] old_repodir = None @@ -78,7 +78,7 @@ def taskrepos(cbtype, *args, **kws): task_id = kws['info']['id'] task = kojihub.Task(task_id) taskinfo = task.getInfo() - if taskinfo['method'] != 'buildArch' or taskinfo['state'] != koji.TASK_STATES['CLOSED']: + if taskinfo['method'] != 'build' or taskinfo['state'] != koji.TASK_STATES['CLOSED']: return task_link = True @@ -86,11 +86,11 @@ def taskrepos(cbtype, *args, **kws): @export_in('host') -def tasksRepoNotifications(task_id, data): +def taskRepoNotifications(task_id, data): if context.opts.get('DisableNotifications'): return email_domain = context.opts['EmailDomain'] email = f"{data['owner']}@{email_domain}" data['task_id'] = task_id - kojihub.make_task('tasksRepoNotifications', [email, data]) + kojihub.make_task('taskRepoNotifications', [email, data]) From cffeab3e96c2beabdc24d436e04fa883c851866f Mon Sep 17 00:00:00 2001 From: Jana Librova Date: Sep 04 2024 07:17:14 +0000 Subject: [PATCH 9/11] Fix review --- diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index d425fc5..a1c7b1a 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -89,22 +89,20 @@ class TaskReposTask(BaseTaskHandler): def build_arches(self, rpms): arches = set() - for rpm in rpms: - if isinstance(rpm, str): - rpm = koji.parse_NVRA(os.path.basename(rpm)) - if rpm['arch'] == 'src': + for rpm, arch in rpms: + if arch == 'src': continue - arches.add(rpm['arch']) + if arch not in arches: + arches.add(arch) return arches def create_pkg_list(self, rpms, repodir): remote_pi = koji.PathInfo(topdir='') dest_dirs = [] pkglist = [] - for rpm in rpms: + for rpm, arch in rpms: url = joinpath(remote_pi.work(), rpm) url = f'toplink{url}' - arch = rpm.split('.')[-2] dest = joinpath(repodir, arch) if dest not in dest_dirs: dest_dirs.append(dest) @@ -127,30 +125,27 @@ class TaskReposTask(BaseTaskHandler): repotype = 'scratch' info = None rpms = [] - results = { - 'rpms': [], - 'srpms': [], - } + srpms = [] + results = [] for taskinfo in tasksinfo: task_result = self.session.getTaskResult(taskinfo['id']) - results['rpms'].extend(task_result['rpms']) - results['srpms'].extend(task_result['srpms']) - if results['srpms'] != []: - srpm = results['srpms'][0] - srpmname = os.path.basename(results['srpms'][0]) - info = koji.parse_NVRA(srpmname) - if info is None: - raise koji.GenericError("SRPM is missing.") - if results['rpms'] != []: - rpms.extend(results['rpms']) + for tr_rpm in task_result['rpms']: + results.append([tr_rpm, taskinfo['arch']]) + if task_result['srpms'] != []: + srpms.extend([task_result['srpms'][0], taskinfo['arch']]) + srpmname = os.path.basename(task_result['srpms'][0]) + info = koji.parse_NVRA(srpmname) + if info is None: + raise koji.GenericError("SRPM is missing.") + if results != []: + rpms.extend(results) repodir = joinpath(self.workdir, info['name'], info['version'], info['release']) repodir_nvr = joinpath(repotype, info['name'], info['version'], info['release']) nvr = '%s-%s-%s' % (info['name'], info['version'], info['release']) koji.ensuredir(repodir) - self.create_pkg_list(rpms, repodir) - rpms.append(srpm) + rpms.append(srpms) for arch in self.build_arches(rpms): self.gen_repodata(joinpath(repodir, arch)) for arch in self.build_arches(rpms): diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py index 6a4ded0..3c3bfe8 100644 --- a/plugins/hub/taskrepos_hub.py +++ b/plugins/hub/taskrepos_hub.py @@ -18,20 +18,6 @@ __all__ = ('taskrepos', 'taskRepoNotifications', 'taskRepoDone') logger = logging.getLogger('koji.hub') -def recursive_walk(dirpath, repo_urls, topurl, task_link): - for root, dir, files in os.walk(dirpath): - if dir: - for subfolder in dir: - recursive_walk(subfolder, repo_urls, topurl, task_link) - if task_link: - folder_name_part = root[root.find('repos-tasks'):] - else: - folder_name_part = root[root.find('tasks'):] - for filename in files: - repo_urls.append(joinpath(topurl, folder_name_part, filename)) - return repo_urls - - @export_in('host') def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): repo_urls = [] @@ -46,6 +32,7 @@ def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) safer_move(src, taskfile) + repo_urls.append(taskfile) if task_link: task_dir_with_id = joinpath( @@ -59,12 +46,18 @@ def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): rmtree(reponvr_dir) os.symlink(task_dir_with_id, reponvr_dir) + for i in range(0, len(repo_urls) - 1): + if task_link: + repo_urls[i] = repo_urls[i].replace( + joinpath(koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), + str(task_id)), reponvr_dir) + repo_urls[i] = repo_urls[i].replace(koji.pathinfo.topdir, topurl) + if task_link: files_dir = reponvr_dir else: files_dir = joinpath( koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), str(task_id)) - repo_urls = recursive_walk(files_dir, repo_urls, topurl, task_link) koji.util.rmtree(joinpath( koji.pathinfo.work(), '/'.join(rp.split(str(task_id))[:1]), str(task_id))) From 28d587334fdadca36210e8c967a35992268112a9 Mon Sep 17 00:00:00 2001 From: Jana Librova Date: Sep 16 2024 08:06:57 +0000 Subject: [PATCH 10/11] Fix review --- diff --git a/plugins/builder/taskrepos.py b/plugins/builder/taskrepos.py index a1c7b1a..8c05f6b 100644 --- a/plugins/builder/taskrepos.py +++ b/plugins/builder/taskrepos.py @@ -214,6 +214,7 @@ module_hotfixes=1 repos.append([uploadpath, fn]) repo_files.append(relpath) return repo_files, repos, repopath_part + #return repopath_part def handler(self, task_id, task_link=False): read_config() @@ -221,16 +222,16 @@ module_hotfixes=1 if not isinstance(task_id, int): task_id = int(task_id) taskinfo = self.session.getTaskInfo(task_id, request=True) - policy_data = { - 'user_id': taskinfo['owner'], - } - self.session.host.assertPolicy('taskrepos', policy_data) if not taskinfo: raise koji.BuildError('Invalid task ID: %s' % task_id) if taskinfo['method'] != 'build': raise koji.BuildError('%s is not a build task' % task_id) if taskinfo['state'] != 2: - raise koji.BuildError('task %s has not completed successfully' % taskinfo['id']) + raise koji.BuildError('task %s has not completed successfully' % task_id) + policy_data = { + 'user_id': taskinfo['owner'], + } + self.session.host.assertPolicy('taskrepos', policy_data) buildarch_subtasks = [subtask for subtask in self.session.getTaskChildren(task_id) if subtask['method'] == 'buildArch'] repodir, repodir_nvr, data['nvr'] = self.create_repos(buildarch_subtasks) @@ -245,7 +246,7 @@ module_hotfixes=1 return [uploadpath, repo_files] -class taskRepoNotifications(BaseTaskHandler): +class TaskRepoNotifications(BaseTaskHandler): Methods = ['taskRepoNotifications'] _taskWeight = 0.1 diff --git a/plugins/hub/taskrepos_hub.py b/plugins/hub/taskrepos_hub.py index 3c3bfe8..c4c94c2 100644 --- a/plugins/hub/taskrepos_hub.py +++ b/plugins/hub/taskrepos_hub.py @@ -46,7 +46,7 @@ def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): rmtree(reponvr_dir) os.symlink(task_dir_with_id, reponvr_dir) - for i in range(0, len(repo_urls) - 1): + for i in range(0, len(repo_urls)): if task_link: repo_urls[i] = repo_urls[i].replace( joinpath(koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), @@ -62,6 +62,8 @@ def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link): koji.util.rmtree(joinpath( koji.pathinfo.work(), '/'.join(rp.split(str(task_id))[:1]), str(task_id))) return repo_urls, files_dir, old_repodir + #return files_dir, old_repodir + #return repo_urls @callback('postTaskStateChange') From a3a25e5771cf44ac8d2cc26c6daf7e6c23613a82 Mon Sep 17 00:00:00 2001 From: Jana Librova Date: Sep 16 2024 08:07:21 +0000 Subject: [PATCH 11/11] Unit tests for taskrepos plugin --- diff --git a/tests/test_plugins/test_taskrepos_builder.py b/tests/test_plugins/test_taskrepos_builder.py new file mode 100644 index 0000000..cbfe002 --- /dev/null +++ b/tests/test_plugins/test_taskrepos_builder.py @@ -0,0 +1,202 @@ +import shutil +import tempfile +import unittest + +import mock +from plugins.builder import taskrepos + +import koji + + +class TestTaskReposTask(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.session = mock.MagicMock() + self.options = mock.MagicMock() + koji.ensuredir = mock.MagicMock() + self.workdir = tempfile.mkdtemp() + self.task = taskrepos.TaskReposTask( + 123, 'taskrepos', {}, self.session, self.options, self.workdir) + self.os_symlink = mock.patch('os.symlink').start() + self.os_makedir = mock.patch('os.makedirs').start() + + def tearDown(self): + shutil.rmtree(self.workdir) + mock.patch.stopall() + + def test_build_arches(self): + rpms = [['rpm1', 'arch1'], ['rpm2', 'arch2'], ['rpm3', 'arch1']] + rv = self.task.build_arches(rpms) + self.assertEqual(rv, {'arch1', 'arch2'}) + + @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data") + def test_create_repo(self, builtins_open): + taskinfo = [{'parent': 2, 'id': 5, 'arch': 'arch1'}, + {'parent': 2, 'id': 6, 'arch': 'arch2'}] + self.session.getTaskResult.side_effect = [{'srpms': ['tasks/5/5/repo_-1.1-11.src.rpm'], + 'rpms': ['tasks/5/5/rpm-1.1-11.arch1.rpm', + 'tasks/5/5/rpm-1.2-11.arch1.rpm']}, + {'srpms': ['tasks/6/6/repo-1.1-11.src.rpm'], + 'rpms': ['tasks/6/6/rpm-1.1-11.arch2.rpm', + 'tasks/6/6/rpm-1.2-11.arch2.rpm']}] + self.task.build_arches = mock.MagicMock() + self.task.build_arches.side_effect = [['arch1', 'arch2'], ['arch1', 'arch2']] + self.task.gen_repodata = mock.MagicMock() + self.task.gen_repodata.return_value = None + self.task.merge_arch_repo = mock.MagicMock() + self.task.merge_arch_repo.return_value = None + rv = self.task.create_repos(taskinfo) + self.assertEqual(rv, + (f'{self.workdir}/repo/1.1/11', 'official/repo/1.1/11', 'repo-1.1-11')) + + @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data") + def test_create_pkg_list(self, builtins_open): + rpms = [['rpm1', 'arch1'], ['rpm2', 'arch2'], ['rpm3', 'arch1']] + rv = self.task.create_pkg_list(rpms, 'path/to/repodir') + self.assertEqual(rv, ['path/to/repodir/arch1', 'path/to/repodir/arch2']) + + @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data") + def test_write_repo_file_non_scratch_noarch(self, builtins_open): + taskinfo = {'request': [{}, 'build-trg-test'], 'id': 1, 'arch': 'arch1'} + buildarch_subtasks = [{'parent': 2, 'id': 5, 'arch': 'arch1', 'label': 'noarch'}, + {'parent': 2, 'id': 6, 'arch': 'arch2', 'label': 'noarch'}] + repodir = 'path/to/repodir/' + nvr = 'rpm-1.1-11' + self.options.topurl = 'https://topurl.com' + rv = self.task.write_repo_file(taskinfo, buildarch_subtasks, repodir, nvr) + self.assertEqual(rv, 'https://topurl.com/repos-tasks/tasks/123/123/noarch/') + + @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data") + def test_write_repo_file_scratch_basearch(self, builtins_open): + taskinfo = {'request': [{'scratch': True}, 'build-trg-test', 'giturl'], + 'id': 1, 'arch': 'arch1'} + buildarch_subtasks = [{'parent': 2, 'id': 5, 'arch': 'arch1', 'label': 'arch1'}, + {'parent': 2, 'id': 6, 'arch': 'arch1', 'label': 'arch1'}] + repodir = 'path/to/repodir/' + nvr = 'rpm-1.1-11' + self.options.topurl = 'https://topurl.com' + rv = self.task.write_repo_file(taskinfo, buildarch_subtasks, repodir, nvr) + self.assertEqual(rv, 'https://topurl.com/repos-tasks/tasks/123/123/$basearch/') + + @mock.patch('subprocess.run') + def test_gen_repodata(self, subprocess_run): + mock_stdout = mock.MagicMock() + mock_stdout.configure_mock( + **{ + "stdout.decode.return_value": '{"A": 3}' + } + ) + subprocess_run.return_value = mock_stdout + self.task.gen_repodata('path/to/repodir/arch') + + @mock.patch('subprocess.run') + @mock.patch("os.listdir", return_value=['rpm-1.1-11.arch1.rpm', 'rpm-1.2-11.arch1.rpm']) + def test_merge_arch_repo_without_srcdir(self, subprocess_run, listdir): + arch = 'arch1' + repodir = 'path/to/repodir/' + srcdir = 'path/to/srcdir/' + mock_stdout = mock.MagicMock() + mock_stdout.configure_mock( + **{ + "stdout.decode.return_value": '{"A": 3}' + } + ) + subprocess_run.return_value = mock_stdout + self.task.merge_arch_repo(arch, repodir, srcdir) + + def test_handler_invalid_task_id(self): + taskrepos.read_config.return_value = {} + self.session.host.assertPolicy = mock.MagicMock() + self.session.host.assertPolicy.return_value = True + self.session.getTaskInfo.return_value = None + with self.assertRaises(koji.GenericError) as ex: + self.task.handler(111, True) + self.assertEqual(str(ex.exception), "Invalid task ID: 111") + + def test_handler_not_build_task(self): + taskrepos.read_config.return_value = {} + self.session.host.assertPolicy = mock.MagicMock() + self.session.host.assertPolicy.return_value = True + self.session.getTaskInfo.return_value = { + 'request': [{}, 'build-trg-test'], 'id': 111, 'arch': 'arch1', 'owner': 'testuser', + 'method': 'buildArch', 'state': 2} + with self.assertRaises(koji.GenericError) as ex: + self.task.handler(111, True) + self.assertEqual(str(ex.exception), "111 is not a build task") + + def test_handler_not_closed_task(self): + taskrepos.read_config.return_value = {} + self.session.host.assertPolicy = mock.MagicMock() + self.session.host.assertPolicy.return_value = True + self.session.getTaskInfo.return_value = { + 'request': [{}, 'build-trg-test'], 'id': 111, 'arch': 'arch1', 'owner': 'testuser', + 'method': 'build', 'state': 1} + with self.assertRaises(koji.GenericError) as ex: + self.task.handler(111, True) + self.assertEqual(str(ex.exception), "task 111 has not completed successfully") + + def test_handler_valid(self): + taskrepos.read_config.return_value = {} + self.session.host.assertPolicy = mock.MagicMock() + self.session.host.assertPolicy.return_value = True + self.session.getTaskInfo.return_value = { + 'request': [{}, 'build-trg-test'], 'id': 111, 'arch': 'arch1', 'owner': 'testuser', + 'method': 'build', 'state': 2} + self.session.getTaskChildren.return_value = [ + {'id': 112, 'arch': 'arch1', 'owner': 'testuser', 'method': 'buildArch', 'state': 2}, + {'id': 113, 'arch': 'arch1', 'owner': 'testuser', 'method': 'tagBuild', 'state': 2}] + self.task.create_repos = mock.MagicMock() + self.task.create_repos.return_value = [ + 'tmp/dir//repo/1.1/11', 'official/repo/1.1/11', 'repo-1.1-11'] + self.task.write_repo_file = mock.MagicMock() + self.task.write_repo_file.return_value = \ + 'https://topurl.com/repos-tasks/tasks/123/123/noarch/' + self.task.upload_repo = mock.MagicMock() + self.task.upload_repo.return_value = (['../../../dir/file'], + [['path/to/repodir//../../../dir', 'file']], + 'path/to/repodir/') + self.session.host.taskRepoDone.return_value = ( + ['https://topurl.com/repos-tasks/tasks/123/123/upload_path_1/fn_1'], + 'topdir/repos-tasks/tasks/123/123', None) + rv = self.task.handler(111, True) + self.assertEqual(rv, (['topdir/repos-tasks/tasks/123/123', ['../../../dir/file']])) + + @mock.patch('os.path.islink') + @mock.patch('os.walk') + def test_upload_repo(self, os_walk, islink): + repodir = 'path/to/repodir/' + os_walk.return_value = (('dir', [], ['file']),) + islink.return_value = False + rv = self.task.upload_repo(repodir) + self.assertEqual(rv, (['../../../dir/file'], + [['path/to/repodir//../../../dir', 'file']], + 'path/to/repodir/')) + + +class TestTaskRepoNotifications(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.session = mock.MagicMock() + self.options = mock.MagicMock() + self.options.from_addr = 'notifuser@domain.com' + self.task_notif = taskrepos.TaskRepoNotifications( + 123, 'taskRepoNotifications', {}, self.session, self.options) + self.SMTP = mock.patch('smtplib.SMTP').start() + + def tearDown(self): + mock.patch.stopall() + + def test_notif(self): + recipient = 'testuser@domain.com' + server = mock.MagicMock() + self.SMTP.return_value = server + taskrepos.read_config.return_value = {'ticketlink': 'https://ticketlink.com', + 'sourcecodelink': 'https://sourcecodelink.com', + 'repo_lifetime': 14, + 'email_template': 'email_template'} + rv = self.task_notif.handler(recipient, { + 'repo_urls': ['rpm-1.1-11.arch.rpm', 'rpm-1.2-11.arch.rpm', 'url/repo/repofile.repo'], + 'nvr': 'rpm-1.2-11', 'old_repodir': None}) + self.assertEqual(rv, 'sent notification of taskrepo 123 to: %s' % recipient) diff --git a/tests/test_plugins/test_taskrepos_hub.py b/tests/test_plugins/test_taskrepos_hub.py new file mode 100644 index 0000000..7d94122 --- /dev/null +++ b/tests/test_plugins/test_taskrepos_hub.py @@ -0,0 +1,141 @@ +import unittest + +import mock +from plugins.hub import taskrepos_hub + +import koji +import kojihub + +QP = kojihub.QueryProcessor + + +class TestTaskrepos(unittest.TestCase): + def getQuery(self, *args, **kwargs): + query = QP(*args, **kwargs) + query.executeOne = self.query_executeone + self.queries.append(query) + return query + + def setUp(self): + self._dml = mock.patch('kojihub.db._dml').start() + self.context = mock.patch('plugins.hub.taskrepos_hub.context').start() + self.context.session.assertPerm = mock.MagicMock() + kojihub.make_task = mock.MagicMock() + kojihub.make_task.return_value = 1 + self.QueryProcessor = mock.patch('kojihub.kojihub.QueryProcessor', + side_effect=self.getQuery).start() + self.queries = [] + self.query_executeone = mock.MagicMock() + + def tearDown(self): + mock.patch.stopall() + + def test_valid(self): + self.query_executeone.return_value = {'method': 'build', 'state': 2} + taskrepos_hub.taskrepos('cbtype', 'test', attribute='state', info={'id': 111}) + kojihub.make_task.assert_called_once_with('taskrepos', [111, True]) + self.assertEqual(len(self.queries), 1) + + def test_attribute_not_state(self): + self.query_executeone.return_value = {'method': 'build', 'state': 2} + taskrepos_hub.taskrepos('cbtype', 'test', attribute='test', info={'id': 111}) + kojihub.make_task.assert_not_called() + self.assertEqual(len(self.queries), 0) + + def test_task_not_build(self): + self.query_executeone.return_value = {'method': 'buildArch', 'state': 2} + taskrepos_hub.taskrepos('cbtype', 'test', attribute='state', info={'id': 111}) + kojihub.make_task.assert_not_called() + self.assertEqual(len(self.queries), 1) + + def test_task_not_closed(self): + self.query_executeone.return_value = {'method': 'build', 'state': 1} + taskrepos_hub.taskrepos('cbtype', 'test', attribute='state', info={'id': 111}) + kojihub.make_task.assert_not_called() + self.assertEqual(len(self.queries), 1) + + +class TestTaskRepoDone(unittest.TestCase): + def setUp(self): + self.maxDiff = None + self.context = mock.patch('plugins.hub.taskrepos_hub.context').start() + self.context.session.assertPerm = mock.MagicMock() + self.pathinfo_work = mock.patch('koji.pathinfo.work').start() + koji.pathinfo.topdir = 'topdir' + self.os_symlink = mock.patch('os.symlink').start() + + def tearDown(self): + mock.patch.stopall() + + @mock.patch("koji.util.rmtree") + @mock.patch('shutil.move') + @mock.patch('koji.ensuredir') + @mock.patch('os.mkdir') + @mock.patch('os.path.exists') + @mock.patch('os.path.isdir') + def test_valid_with_task_link(self, mock_isdir, mock_exists, mock_mkdir, ensuredir, + shutil_move, rmtree): + mock_exists.side_effect = [True, False, False] + mock_isdir.return_value = True + self.pathinfo_work.return_value = '/' + rv = taskrepos_hub.taskRepoDone(123, [['repo/dir/nvr/1.1/upload_path_1', 'fn_1']], + 'repo/dir/nvr/1.1', 'https://topurl.com', True) + self.assertEqual(rv, ( + ['https://topurl.com/repos-tasks/repo/dir/nvr/1.1/upload_path_1/fn_1'], + 'topdir/repos-tasks/repo/dir/nvr/1.1', None)) + + @mock.patch("koji.util.rmtree") + @mock.patch('shutil.move') + @mock.patch('koji.ensuredir') + @mock.patch('os.mkdir') + @mock.patch('os.path.exists') + @mock.patch('os.path.isdir') + def test_valid_without_task_link(self, mock_isdir, mock_exists, mock_mkdir, ensuredir, + shutil_move, rmtree): + mock_exists.side_effect = [True, False, False] + mock_isdir.return_value = True + self.pathinfo_work.return_value = '/' + rv = taskrepos_hub.taskRepoDone(123, [['repo/dir/nvr/1.1/upload_path_1', 'fn_1']], + 'repo/dir/nvr/1.1', 'https://topurl.com', False) + self.assertEqual(rv, ( + ['https://topurl.com/repos-tasks/tasks/123/123/upload_path_1/fn_1'], + 'topdir/repos-tasks/tasks/123/123', None)) + + @mock.patch('koji.ensuredir') + @mock.patch('os.path.exists') + def test_uploaded_file_missing(self, mock_exists, ensuredir): + mock_exists.return_value = False + self.pathinfo_work.return_value = '/' + with self.assertRaises(koji.GenericError) as ex: + taskrepos_hub.taskRepoDone(123, [['repo/dir/nvr/1.1/upload_path_1', 'fn_1']], + 'repo/dir/nvr/1.1', 'https://topurl.com', True) + self.assertEqual(str(ex.exception), + "uploaded file missing: /repo/dir/nvr/1.1/upload_path_1/fn_1") + + +class TestTaskReposNotifications(unittest.TestCase): + def setUp(self): + self.context = mock.patch('plugins.hub.taskrepos_hub.context').start() + self.context.session.assertPerm = mock.MagicMock() + kojihub.make_task = mock.MagicMock() + kojihub.make_task.return_value = 1 + self.context.opts = {'EmailDomain': 'testdomain.com'} + + def tearDown(self): + mock.patch.stopall() + + def test_taskrepoNotification(self): + self.context.opts['DisableNotifications'] = False + taskrepos_hub.taskRepoNotifications(123, {'owner': 'testuser', 'task_id': 112}) + kojihub.make_task.assert_called_once_with( + 'taskRepoNotifications', + [ + 'testuser@testdomain.com', + {'owner': 'testuser', 'task_id': 123} + ], + ) + + def test_taskrepoNotification_disabled(self): + self.context.opts['DisableNotifications'] = True + taskrepos_hub.taskRepoNotifications(123, {'owner': 'testuser', 'task_id': 112}) + kojihub.make_task.assert_not_called()