From 7653dedd6974f5dbf94b9b812f6305f77d95748c Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: May 31 2017 07:55:50 +0000 Subject: [PATCH 1/2] Refactor code to re-organize handlers Change the code structure to have one handler for each event, after the change, a handler will only handle one type of events. For example MBSModuleStateChangeHandler in handlers/mbs/module_state_change.py now only handles the event of MBSModuleStateChangeEvent. Updated names of parsers, events and handlers to keep consistant style of naming. All parser names ended with 'Event', and 'Parser' for parsers, 'Handler' for handlers. Updated test cases per the changes. --- diff --git a/conf/config.py b/conf/config.py index dcdecd5..26e85bc 100644 --- a/conf/config.py +++ b/conf/config.py @@ -42,10 +42,12 @@ class BaseConfiguration(object): # List of enabled composing handlers. HANDLERS = [ - "freshmaker.handlers.mbs:MBS", # Module Build Service - "freshmaker.handlers.image_builder:DockerImageRebuildHandler", - "freshmaker.handlers.image_builder:DockerImageRebuildHandlerForBodhi", - "freshmaker.handlers.buildsys:BuildsysHandler", + "freshmaker.handlers.bodhi:BodhiUpdateCompleteStableHandler", + "freshmaker.handlers.git:GitDockerfileChangeHandler", + "freshmaker.handlers.git:GitModuleMetadataChangeHandler", + "freshmaker.handlers.git:GitRPMSpecChangeHandler", + "freshmaker.handlers.koji:KojiTaskStateChangeHandler", + "freshmaker.handlers.mbs:MBSModuleStateChangeHandler", ] # Base URL of git repository with source artifacts. diff --git a/freshmaker/config.py b/freshmaker/config.py index 0ed824a..8c0e2c1 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -134,7 +134,7 @@ class Config(object): 'desc': 'Global network retry interval for read/write operations, in seconds.'}, 'handlers': { 'type': list, - 'default': ["freshmaker.handlers.mbs:MBS"], + 'default': ["freshmaker.handlers.mbs:MBSModuleStateChangeHandler"], 'desc': 'List of enabled handlers.'}, 'git_base_url': { 'type': str, diff --git a/freshmaker/consumer.py b/freshmaker/consumer.py index 0e7ef38..9c81a4b 100644 --- a/freshmaker/consumer.py +++ b/freshmaker/consumer.py @@ -28,10 +28,10 @@ to use. import fedmsg.consumers import moksha.hub import freshmaker.handlers -import freshmaker.parsers.mbsmodule -import freshmaker.parsers.gitreceive -import freshmaker.parsers.bodhiupdate -import freshmaker.parsers.buildsys +import freshmaker.parsers.mbs +import freshmaker.parsers.git +import freshmaker.parsers.bodhi +import freshmaker.parsers.koji from freshmaker import log, conf, messaging, events @@ -63,10 +63,10 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): self.incoming.put(msg) def register_parsers(self): - events.BaseEvent.register_parser(freshmaker.parsers.mbsmodule.MBSModuleParser) - events.BaseEvent.register_parser(freshmaker.parsers.gitreceive.GitReceiveParser) - events.BaseEvent.register_parser(freshmaker.parsers.bodhiupdate.UpdateCompleteStableParser) - events.BaseEvent.register_parser(freshmaker.parsers.buildsys.BuildsysParser) + events.BaseEvent.register_parser(freshmaker.parsers.bodhi.BodhiUpdateCompleteStableParser) + events.BaseEvent.register_parser(freshmaker.parsers.git.GitReceiveParser) + events.BaseEvent.register_parser(freshmaker.parsers.koji.KojiTaskStateChangeParser) + events.BaseEvent.register_parser(freshmaker.parsers.mbs.MBSModuleStateChangeParser) log.debug("Parser classes: %r", events.BaseEvent._parsers) self.topic = events.BaseEvent.get_parsed_topics() diff --git a/freshmaker/events.py b/freshmaker/events.py index 1bc3535..7fd9a48 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -122,35 +122,35 @@ class BaseEvent(object): return None -class ModuleBuilt(BaseEvent): +class MBSModuleStateChangeEvent(BaseEvent): """ A class that inherits from BaseEvent to provide an event object for a module event generated by module-build-service :param msg_id: the id of the msg (e.g. 2016-SomeGUID) :param module_build_id: the id of the module build :param module_build_state: the state of the module build """ - def __init__(self, msg_id, module_build_id, module_build_state, name, stream): - super(ModuleBuilt, self).__init__(msg_id) - self.module_build_id = module_build_id - self.module_build_state = module_build_state - self.module_name = name - self.module_stream = stream + def __init__(self, msg_id, module, stream, build_id, build_state): + super(MBSModuleStateChangeEvent, self).__init__(msg_id) + self.module = module + self.stream = stream + self.build_id = build_id + self.build_state = build_state -class ModuleMetadataUpdated(BaseEvent): +class GitModuleMetadataChangeEvent(BaseEvent): """ Provides an event object for "Module metadata in dist-git updated". :param scm_url: SCM URL of a updated module. :param branch: Branch of updated module. """ - def __init__(self, msg_id, scm_url, name, branch): - super(ModuleMetadataUpdated, self).__init__(msg_id) - self.scm_url = scm_url - self.name = name + def __init__(self, msg_id, module, branch, rev): + super(GitModuleMetadataChangeEvent, self).__init__(msg_id) + self.module = module self.branch = branch + self.rev = rev -class RPMSpecUpdated(BaseEvent): +class GitRPMSpecChangeEvent(BaseEvent): """ Provides an event object for "RPM spec file in dist-git updated". @@ -159,7 +159,7 @@ class RPMSpecUpdated(BaseEvent): :param rev: revision. """ def __init__(self, msg_id, rpm, branch, rev): - super(RPMSpecUpdated, self).__init__(msg_id) + super(GitRPMSpecChangeEvent, self).__init__(msg_id) self.rpm = rpm self.branch = branch self.rev = rev @@ -173,19 +173,17 @@ class TestingEvent(BaseEvent): super(TestingEvent, self).__init__(msg_id) -class DockerfileChanged(BaseEvent): +class GitDockerfileChangeEvent(BaseEvent): """Represent the message omitted when Dockerfile is changed in a push""" - def __init__(self, msg_id, repo_url, namespace, repo, branch, rev): - super(DockerfileChanged, self).__init__(msg_id) - self.repo_url = repo_url + def __init__(self, msg_id, container, branch, rev): + super(GitDockerfileChangeEvent, self).__init__(msg_id) + self.container = container self.branch = branch - self.namespace = namespace - self.repo = repo self.rev = rev -class BodhiUpdateCompleteStable(BaseEvent): +class BodhiUpdateCompleteStableEvent(BaseEvent): """Event when RPMs are available in Fedora master mirrors Refer to an example in datagrepper: @@ -207,17 +205,17 @@ class BodhiUpdateCompleteStable(BaseEvent): branch. Refer to the example given above to see all available attributes in a message. """ - super(BodhiUpdateCompleteStable, self).__init__(msg_id) + super(BodhiUpdateCompleteStableEvent, self).__init__(msg_id) self.update_id = update_id self.builds = builds self.release = release -class KojiTaskStateChanged(BaseEvent): +class KojiTaskStateChangeEvent(BaseEvent): """ Provides an event object for "the state of task changed in koji" """ def __init__(self, msg_id, task_id, task_state): - super(KojiTaskStateChanged, self).__init__(msg_id) + super(KojiTaskStateChangeEvent, self).__init__(msg_id) self.task_id = task_id self.task_state = task_state diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 460179f..1e71927 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -26,6 +26,8 @@ import re import fedmsg.utils from freshmaker import conf, log, db, models +from freshmaker.mbs import MBS +from freshmaker.kojiservice import koji_service def load_handlers(): @@ -61,6 +63,43 @@ class BaseHandler(object): """ raise NotImplementedError() + def build_module(self, name, branch, rev): + """ + Build a module in MBS. + + :param name: module name. + :param branch: module branch. + :param rev: git revision. + """ + mbs = MBS(conf) + return mbs.build_module(name, branch, rev) + + def build_container(self, name, branch, rev): + """ + Build a container in koji. + + :param container: container name. + :param branch: container branch. + :param rev: revision. + """ + with koji_service(profile=conf.koji_profile, logger=log) as service: + log.debug('Logging into {0} with Kerberos authentication.'.format(service.server)) + proxyuser = conf.koji_build_owner if conf.koji_proxyuser else None + service.krb_login(proxyuser=proxyuser) + + if not service.logged_in: + log.error('Could not login server %s', service.server) + return None + + build_source = "{}/container/{}.git?#{}".format(conf.git_base_url, name, rev) + + log.debug('Building container from source: %s', build_source) + + return service.build_container(build_source, + branch, + namespace='container', + scratch=conf.koji_container_scratch_build) + def record_build(self, event, name, type, build_id, dep_of=None): """ Record build in db. diff --git a/freshmaker/handlers/bodhi/__init__.py b/freshmaker/handlers/bodhi/__init__.py new file mode 100644 index 0000000..9b396d6 --- /dev/null +++ b/freshmaker/handlers/bodhi/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .update_complete_stable import BodhiUpdateCompleteStableHandler # noqa diff --git a/freshmaker/handlers/bodhi/update_complete_stable.py b/freshmaker/handlers/bodhi/update_complete_stable.py new file mode 100644 index 0000000..24551af --- /dev/null +++ b/freshmaker/handlers/bodhi/update_complete_stable.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +from itertools import chain + +from freshmaker import conf +from freshmaker import log +from freshmaker import utils +from freshmaker.handlers import BaseHandler +from freshmaker.events import BodhiUpdateCompleteStableEvent +from freshmaker.pdc import PDC +from freshmaker.kojiservice import koji_service + + +class BodhiUpdateCompleteStableHandler(BaseHandler): + """Rebuild docker images when RPMs are synced by Bodhi""" + name = 'BodhiUpdateCompleteStableHandler' + + def can_handle(self, event): + return isinstance(event, BodhiUpdateCompleteStableEvent) + + def handle(self, event): + log.info('Rebuild docker images for event %s, msgid: %s', + BodhiUpdateCompleteStableEvent.__name__, event.msg_id) + + rpms = self.get_rpms_included_in_bodhi_update(event.builds) + containers = self.get_containers_including_rpms(rpms) + + log.info('Found docker images to rebuild: %s', containers) + + for container in containers: + if not self.allow_build(event, 'image', container['name'], container['branch']): + log.info("Skip rebuild of image %s:%s as it's not allowed by configured whitelist/blacklist", + container['name'], container['branch']) + continue + try: + name = container['name'] + branch = container['branch'] + repo_url = '{}/{}/{}'.format(conf.git_base_url, 'container', name) + rev = utils.get_commit_hash(repo_url, branch=branch, logger=log) + + task_id = self.build_container(name, branch, rev) + if task_id is not None: + self.record_build(event, container['name'], 'image', task_id) + except: + log.exception('Error when rebuild %s', container) + + return [] + + def get_rpms_included_in_bodhi_update(self, builds): + build_nvrs = (build['nvr'] for build in builds) + with koji_service(profile=conf.koji_profile, logger=log) as service: + return chain(*[service.get_build_rpms(nvr) for nvr in build_nvrs]) + + def get_containers_including_rpms(self, rpms): + containers = {} + pdc = PDC(conf) + for rpm in rpms: + found = pdc.find_containers_by_rpm_name(rpm['name']) + for container in found: + id = container['id'] + if id not in containers: + container_detail = pdc.get_release_component_by_id(id) + container['branch'] = container_detail['dist_git_branch'] + containers[id] = container + + return containers.values() diff --git a/freshmaker/handlers/buildsys.py b/freshmaker/handlers/buildsys.py deleted file mode 100644 index f17937c..0000000 --- a/freshmaker/handlers/buildsys.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from freshmaker import log, db, models -from freshmaker.handlers import BaseHandler -from freshmaker.events import KojiTaskStateChanged - - -class BuildsysHandler(BaseHandler): - name = "BuildsysHandler" - - def can_handle(self, event): - if isinstance(event, KojiTaskStateChanged): - return True - - return False - - def handle_koji_task_state_changed(self, event): - task_id = event.task_id - task_state = event.task_state - - # check whether the task exists in db as image build - builds = db.session.query(models.ArtifactBuild).filter_by(build_id=task_id, - type=models.ARTIFACT_TYPES['image']).all() - if len(builds) > 1: - raise RuntimeError("Found duplicate image build '%s' in db" % task_id) - if len(builds) == 1: - build = builds[0] - if task_state in ['CLOSED', 'FAILED']: - log.info("Image build '%s' state changed in koji, updating it in db.", task_id) - if task_state == 'CLOSED': - build.state = models.BUILD_STATES['done'] - db.session.commit() - if task_state == 'FAILED': - build.state = models.BUILD_STATES['failed'] - db.session.commit() - - return [] - - def handle(self, event): - if isinstance(event, KojiTaskStateChanged): - return self.handle_koji_task_state_changed(event) - - return [] diff --git a/freshmaker/handlers/git/__init__.py b/freshmaker/handlers/git/__init__.py new file mode 100644 index 0000000..247e34f --- /dev/null +++ b/freshmaker/handlers/git/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .dockerfile_change import GitDockerfileChangeHandler # noqa +from .module_metadata_change import GitModuleMetadataChangeHandler # noqa +from .rpm_spec_change import GitRPMSpecChangeHandler # noqa diff --git a/freshmaker/handlers/git/dockerfile_change.py b/freshmaker/handlers/git/dockerfile_change.py new file mode 100644 index 0000000..e9e010f --- /dev/null +++ b/freshmaker/handlers/git/dockerfile_change.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +from freshmaker import log +from freshmaker.handlers import BaseHandler +from freshmaker.events import GitDockerfileChangeEvent + + +class GitDockerfileChangeHandler(BaseHandler): + name = 'GitDockerfileChangeHandler' + + def can_handle(self, event): + return isinstance(event, GitDockerfileChangeEvent) + + def handle(self, event): + """Rebuild docker image""" + import koji + + log.info('Start to rebuild docker image %s.', event.container) + + if not self.allow_build(event, 'image', event.container, event.branch): + log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", + event.container, event.branch) + return [] + + try: + task_id = self.build_container(event.container, event.branch, event.rev) + + if task_id is not None: + self.record_build(event, event.container, 'image', task_id) + + except koji.krbV.Krb5Error as e: + log.exception('Failed to login Koji via Kerberos using GSSAPI. %s', e.args[1]) + except: + log.exception('Could not create task to build docker image %s', event.container) + + return [] diff --git a/freshmaker/handlers/git/module_metadata_change.py b/freshmaker/handlers/git/module_metadata_change.py new file mode 100644 index 0000000..94967f0 --- /dev/null +++ b/freshmaker/handlers/git/module_metadata_change.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + + +from freshmaker import log +from freshmaker.handlers import BaseHandler +from freshmaker.events import GitModuleMetadataChangeEvent + + +class GitModuleMetadataChangeHandler(BaseHandler): + name = "GitModuleMetadataChangeHandler" + + def can_handle(self, event): + if isinstance(event, GitModuleMetadataChangeEvent): + return True + + return False + + def handle(self, event): + log.info("Triggering rebuild of module %s:%s, metadata updated (%s).", + event.module, event.branch, event.rev) + if not self.allow_build(event, 'module', event.module, event.branch): + log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", + event.module, event.branch) + return [] + + build_id = self.build_module(event.module, event.branch, event.rev) + if build_id is not None: + self.record_build(event, event.module, 'module', build_id) + + return [] diff --git a/freshmaker/handlers/git/rpm_spec_change.py b/freshmaker/handlers/git/rpm_spec_change.py new file mode 100644 index 0000000..123074d --- /dev/null +++ b/freshmaker/handlers/git/rpm_spec_change.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from freshmaker import log, conf, utils +from freshmaker.pdc import PDC +from freshmaker.handlers import BaseHandler +from freshmaker.events import GitRPMSpecChangeEvent + + +class GitRPMSpecChangeHandler(BaseHandler): + name = "GitRPMSpecChangeHandler" + + def can_handle(self, event): + if isinstance(event, GitRPMSpecChangeEvent): + return True + + return False + + def handle(self, event): + """ + Rebuild module when spec file of rpm in module is updated. + """ + rpm = event.rpm + branch = event.branch + rev = event.rev + + log.info("Triggering rebuild of modules on event of rpm (%s:%s) spec updated, rev: %s.", + rpm, branch, rev) + + pdc = PDC(conf) + modules = pdc.get_latest_modules(component_name=rpm, + component_branch=branch, + active='true') + + for module in modules: + name = module['variant_name'] + version = module['variant_version'] + if not self.allow_build(event, 'module', name, version): + log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", + name, version) + continue + log.info("Going to rebuild module '%s:%s'.", name, version) + commit_msg = "Bump to rebuild because of %s rpm spec update (%s)." % (rpm, rev) + rev = utils.bump_distgit_repo('modules', name, branch=version, commit_msg=commit_msg, logger=log) + build_id = self.build_module(name, version, rev) + if build_id is not None: + self.record_build(event, name, 'module', build_id) + + return [] diff --git a/freshmaker/handlers/image_builder.py b/freshmaker/handlers/image_builder.py deleted file mode 100644 index 63b674f..0000000 --- a/freshmaker/handlers/image_builder.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -import os - -from itertools import chain - -import freshmaker.pdc as pdc - -from freshmaker import log, conf -from freshmaker.handlers import BaseHandler -from freshmaker.events import DockerfileChanged -from freshmaker.events import BodhiUpdateCompleteStable -from freshmaker.utils import temp_dir -from freshmaker.utils import _run_command -from freshmaker.utils import get_commit_hash -from freshmaker.kojiservice import koji_service - - -class DockerImageRebuildHandler(BaseHandler): - name = 'DockerImageRebuildHandler' - - def can_handle(self, event): - return isinstance(event, DockerfileChanged) - - def handle(self, event): - """Rebuild docker image""" - import koji - - log.info('Start to rebuild docker image %s', event.repo) - - if not self.allow_build(event, 'image', event.repo, event.branch): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", - event.repo, event.branch) - return [] - - try: - task_id = self.build_image(repo_url=event.repo_url, - rev=event.rev, - branch=event.branch, - namespace=event.namespace) - - self.record_build(event, event.repo, 'image', task_id) - - except koji.krbV.Krb5Error as e: - log.exception('Failed to login Koji via Kerberos using GSSAPI. %s', e.args[1]) - except: - log.exception('Could not create task to build docker image %s', event.repo) - - return [] - - def build_image(self, repo_url, rev, branch, namespace=None): - with koji_service(profile=conf.koji_profile, logger=log) as service: - log.debug('Logging into {0} with Kerberos authentication.'.format(service.server)) - proxyuser = conf.koji_build_owner if conf.koji_proxyuser else None - service.krb_login(proxyuser=proxyuser) - - if not service.logged_in: - log.error('Could not login server %s', service.server) - return - - build_source = '{}?#{}'.format(repo_url, rev) - - log.debug('Build from source: %s', build_source) - - return service.build_container(build_source, - branch, - namespace=namespace, - scratch=conf.koji_container_scratch_build) - - -class DockerImageRebuildHandlerForBodhi(DockerImageRebuildHandler): - """Rebuild docker images when RPMs are synced by Bodhi""" - name = 'DockerImageRebuildForBodhiHandler' - - def __init__(self): - self.pdc_session = pdc.get_client_session(conf) - - def can_handle(self, event): - return isinstance(event, BodhiUpdateCompleteStable) - - def handle(self, event): - log.info('Rebuild docker images for event %s, msgid: %s', - BodhiUpdateCompleteStable.__name__, event.msg_id) - - rpms = self.get_rpms_included_in_bodhi_update(event.builds) - containers = self.get_containers_including_rpms(rpms) - - log.info('Found docker images to rebuild: %s', containers) - - for container in containers: - if not self.allow_build(event, 'image', container['name'], container['branch']): - log.info("Skip rebuild of image %s:%s as it's not allowed by configured whitelist/blacklist", - container['name'], container['branch']) - continue - try: - task_id = self.handle_image_build(container) - self.record_build(event, container['name'], 'image', task_id) - except: - log.exception('Error when rebuild %s', container) - - def handle_image_build(self, container_info): - name = container_info['name'] - branch = container_info['branch'] - repo_url = '{}/{}/{}'.format(conf.git_base_url, 'container', name) - - log.info('Start to rebuild docker image %s from branch %s', name, branch) - - with temp_dir(suffix='-rebuild-docker-image') as working_dir: - self.clone_repository(repo_url, branch, working_dir) - - last_commit_hash = get_commit_hash( - os.path.join(working_dir, name)) - - return self.build_image(repo_url=repo_url, - branch=branch, - rev=last_commit_hash) - - def clone_repository(self, url, branch, working_dir): - cmd = ['git', 'clone', '-b', branch, url] - log.debug('Clone repository: %s', cmd) - _run_command(cmd, rundir=working_dir) - - def get_rpms_included_in_bodhi_update(self, builds): - build_nvrs = (build['nvr'] for build in builds) - with koji_service(profile=conf.koji_profile, logger=log) as service: - return chain(*[service.get_build_rpms(nvr) for nvr in build_nvrs]) - - def get_containers_including_rpms(self, rpms): - containers = {} - for rpm in rpms: - found = pdc.find_containers_by_rpm_name(self.pdc_session, rpm['name']) - for container in found: - id = container['id'] - if id not in containers: - container_detail = pdc.get_release_component(self.pdc_session, id) - container['branch'] = container_detail['dist_git_branch'] - containers[id] = container - - return containers.values() diff --git a/freshmaker/handlers/koji/__init__.py b/freshmaker/handlers/koji/__init__.py new file mode 100644 index 0000000..dbb4f12 --- /dev/null +++ b/freshmaker/handlers/koji/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .task_state_change import KojiTaskStateChangeHandler # noqa diff --git a/freshmaker/handlers/koji/task_state_change.py b/freshmaker/handlers/koji/task_state_change.py new file mode 100644 index 0000000..39d676d --- /dev/null +++ b/freshmaker/handlers/koji/task_state_change.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from freshmaker import log, db, models +from freshmaker.handlers import BaseHandler +from freshmaker.events import KojiTaskStateChangeEvent + + +class KojiTaskStateChangeHandler(BaseHandler): + name = "KojiTaskStateChangeHandler" + + def can_handle(self, event): + if isinstance(event, KojiTaskStateChangeEvent): + return True + + return False + + def handle(self, event): + task_id = event.task_id + task_state = event.task_state + + # check whether the task exists in db as image build + builds = db.session.query(models.ArtifactBuild).filter_by(build_id=task_id, + type=models.ARTIFACT_TYPES['image']).all() + if len(builds) > 1: + raise RuntimeError("Found duplicate image build '%s' in db" % task_id) + if len(builds) == 1: + build = builds.pop() + if task_state in ['CLOSED', 'FAILED']: + log.info("Image build '%s' state changed in koji, updating it in db.", task_id) + if task_state == 'CLOSED': + build.state = models.BUILD_STATES['done'] + db.session.commit() + if task_state == 'FAILED': + build.state = models.BUILD_STATES['failed'] + db.session.commit() + + return [] diff --git a/freshmaker/handlers/mbs.py b/freshmaker/handlers/mbs.py deleted file mode 100644 index 3791e05..0000000 --- a/freshmaker/handlers/mbs.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - -import requests - -from freshmaker import log, conf, utils, pdc, db, models -from freshmaker.handlers import BaseHandler -from freshmaker.events import ModuleBuilt, ModuleMetadataUpdated, RPMSpecUpdated - - -MBS_BUILD_STATES = { - "init": 0, - "wait": 1, - "build": 2, - "done": 3, - "failed": 4, - "ready": 5, -} - - -class MBS(BaseHandler): - name = "MBS" - - def can_handle(self, event): - # TODO: Handle only when something depends on - # this module. - if isinstance(event, ModuleBuilt): - return True - - if isinstance(event, ModuleMetadataUpdated): - return True - - if isinstance(event, RPMSpecUpdated): - return True - - return False - - def rebuild_module(self, scm_url, branch): - """ - Rebuilds the module defined by scm_url and branch in MBS. - Returns build id or None in case of error. - """ - headers = {} - headers["Authorization"] = "Bearer %s" % conf.mbs_auth_token - - body = {'scmurl': scm_url, 'branch': branch} - url = "%s/module-build-service/1/module-builds/" % conf.mbs_base_url - - resp = requests.request("POST", url, headers=headers, json=body) - data = resp.json() - if 'id' in data: - log.info("Triggered rebuild of %s, MBS build_id=%s", scm_url, data['id']) - return data['id'] - else: - log.error("Error when triggering rebuild of %s: %s", scm_url, data) - return None - - def bump_and_rebuild_module(self, name, branch, commit_msg=None): - """Bump module repo with an empty commit and submit a module build request to MBS""" - commitid = None - with utils.temp_dir(prefix='freshmaker-%s-' % name) as repodir: - try: - utils.clone_module_repo(name, repodir, branch=branch, user=conf.git_user, logger=log) - msg = commit_msg or "Bump" - utils.add_empty_commit(repodir, msg=msg, logger=log) - commitid = utils.get_commit_hash(repodir) - utils.push_repo(repodir, logger=log) - except Exception: - log.exception("Failed to update module repo of '%s-%s'.", name, branch) - - if commitid is not None: - scm_url = conf.git_base_url + '/modules/%s.git?#%s' % (name, commitid) - return self.rebuild_module(scm_url, branch) - - def handle_metadata_update(self, event): - log.info("Triggering rebuild of %s, metadata updated", event.scm_url) - if not self.allow_build(event, 'module', event.name, event.branch): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", - event.name, event.branch) - return [] - build_id = self.rebuild_module(event.scm_url, event.branch) - if build_id is not None: - self.record_build(event, event.name, 'module', build_id) - - return [] - - def handle_module_built(self, event): - """ - When there is any module built and state is 'ready', query PDC to get - all modules that depends on this module, rebuild all these depending - modules. - """ - module_name = event.module_name - module_stream = event.module_stream - build_id = event.module_build_id - build_state = event.module_build_state - - # update build state if the build is submitted by Freshmaker - builds = db.session.query(models.ArtifactBuild).filter_by(build_id=build_id, - type=models.ARTIFACT_TYPES['module']).all() - if len(builds) > 1: - raise RuntimeError("Found duplicate module build '%s' in db" % build_id) - if len(builds) == 1: - build = builds[0] - if build_state in [MBS_BUILD_STATES['ready'], MBS_BUILD_STATES['failed']]: - log.info("Module build '%s' state changed in MBS, updating it in db.", build_id) - if build_state == MBS_BUILD_STATES['ready']: - build.state = models.BUILD_STATES['done'] - if build_state == MBS_BUILD_STATES['failed']: - build.state = models.BUILD_STATES['failed'] - db.session.commit() - - # Rebuild depending modules when state of ModuleBuilt is 'ready' - if build_state == MBS_BUILD_STATES['ready']: - log.info("Triggering rebuild of modules depending on %s:%s " - "in MBS", module_name, module_stream) - - pdc_session = pdc.get_client_session(conf) - modules = pdc.get_latest_modules(pdc_session, - build_dep_name=module_name, - build_dep_stream=module_stream, - active='true') - for mod in modules: - name = mod['variant_name'] - version = mod['variant_version'] - if not self.allow_build(event, 'module', name, version): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", - name, version) - continue - commit_msg = "Bump to rebuild because of %s update" % module_name - build_id = self.bump_and_rebuild_module(name, version, commit_msg=commit_msg) - if build_id is not None: - self.record_build(event, name, 'module', build_id) - - return [] - - def handle_rpm_spec_updated(self, event): - """ - Rebuild module when spec file of rpm in module is updated. - """ - rpm = event.rpm - branch = event.branch - rev = event.rev - - log.info("Triggering rebuild of modules on event of rpm (%s:%s) spec updated, rev: %s.", - rpm, branch, rev) - - pdc_session = pdc.get_client_session(conf) - modules = pdc.get_latest_modules(pdc_session, - component_name=rpm, - component_branch=branch, - active='true') - for mod in modules: - module_name = mod['variant_name'] - module_branch = mod['variant_version'] - if not self.allow_build(event, 'module', module_name, module_branch): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", - module_name, module_branch) - continue - log.info("Going to rebuild module '%s:%s'.", module_name, module_branch) - commit_msg = "Bump to rebuild because of %s rpm spec update (%s)." % (rpm, rev) - build_id = self.bump_and_rebuild_module(module_name, module_branch, commit_msg=commit_msg) - if build_id is not None: - self.record_build(event, module_name, 'module', build_id) - - return [] - - def handle(self, event): - if isinstance(event, ModuleMetadataUpdated): - return self.handle_metadata_update(event) - elif isinstance(event, ModuleBuilt): - return self.handle_module_built(event) - elif isinstance(event, RPMSpecUpdated): - return self.handle_rpm_spec_updated(event) - - return [] diff --git a/freshmaker/handlers/mbs/__init__.py b/freshmaker/handlers/mbs/__init__.py new file mode 100644 index 0000000..eb95d91 --- /dev/null +++ b/freshmaker/handlers/mbs/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .module_state_change import MBSModuleStateChangeHandler # noqa diff --git a/freshmaker/handlers/mbs/module_state_change.py b/freshmaker/handlers/mbs/module_state_change.py new file mode 100644 index 0000000..b333755 --- /dev/null +++ b/freshmaker/handlers/mbs/module_state_change.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from freshmaker import log, conf, utils, db, models +from freshmaker.mbs import MBS +from freshmaker.pdc import PDC +from freshmaker.handlers import BaseHandler +from freshmaker.events import MBSModuleStateChangeEvent + + +class MBSModuleStateChangeHandler(BaseHandler): + name = "MBSModuleStateChangeHandler" + + def can_handle(self, event): + if isinstance(event, MBSModuleStateChangeEvent): + return True + + return False + + def handle(self, event): + """ + Update build state in db when module state changed in MBS and the + build is submitted by Freshmaker (can find that build in db). If + build state is 'ready', query PDC to get all modules that depends + on this module, rebuild all these depending modules. + """ + module_name = event.module + module_stream = event.stream + build_id = event.build_id + build_state = event.build_state + + # update build state if the build is submitted by Freshmaker + builds = db.session.query(models.ArtifactBuild).filter_by(build_id=build_id, + type=models.ARTIFACT_TYPES['module']).all() + if len(builds) > 1: + raise RuntimeError("Found duplicate module build '%s' in db" % build_id) + if len(builds) == 1: + build = builds.pop() + if build_state in [MBS.BUILD_STATES['ready'], MBS.BUILD_STATES['failed']]: + log.info("Module build '%s' state changed in MBS, updating it in db.", build_id) + if build_state == MBS.BUILD_STATES['ready']: + build.state = models.BUILD_STATES['done'] + if build_state == MBS.BUILD_STATES['failed']: + build.state = models.BUILD_STATES['failed'] + db.session.commit() + + # Rebuild depending modules when state of MBSModuleStateChangeEvent is 'ready' + if build_state == MBS.BUILD_STATES['ready']: + log.info("Triggering rebuild of modules depending on %s:%s " + "in MBS", module_name, module_stream) + + pdc = PDC(conf) + modules = pdc.get_latest_modules(build_dep_name=module_name, + build_dep_stream=module_stream, + active='true') + + for mod in modules: + name = mod['variant_name'] + version = mod['variant_version'] + if not self.allow_build(event, 'module', name, version): + log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist/blacklist", + name, version) + continue + # bump module repo first + commit_msg = "Bump to rebuild because of %s update" % module_name + rev = utils.bump_distgit_repo('modules', name, branch=version, commit_msg=commit_msg, logger=log) + build_id = self.build_module(name, version, rev) + if build_id is not None: + self.record_build(event, name, 'module', build_id) + + return [] diff --git a/freshmaker/mbs.py b/freshmaker/mbs.py new file mode 100644 index 0000000..b58a76d --- /dev/null +++ b/freshmaker/mbs.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +import requests + +from freshmaker import log + + +class MBS(object): + BUILD_STATES = { + "init": 0, + "wait": 1, + "build": 2, + "done": 3, + "failed": 4, + "ready": 5, + } + + def __init__(self, config): + """ + :param config: config which has mbs_base_url, mbs_auth_token and git_base_url. + can just be an instance of freshmaker.config.Config + """ + self.base_url = config.mbs_base_url + self.auth_token = config.mbs_auth_token + self.git_base_url = config.git_base_url + + def build_module(self, name, branch, rev): + """ + Build module defined by name, branch and rev in MBS. + + :param name: module name. + :param branch: module branch. + :param rev: git revision. + :return: Build id or None in case of error. + """ + scm_url = self.git_base_url + '/modules/%s.git?#%s' % (name, rev) + + headers = {} + headers["Authorization"] = "Bearer %s" % self.auth_token + + body = {'scmurl': scm_url, 'branch': branch} + url = "%s/module-build-service/1/module-builds/" % self.base_url + + resp = requests.request("POST", url, headers=headers, json=body) + data = resp.json() + if 'id' in data: + log.info("Triggered build of %s, MBS build_id=%s", scm_url, data['id']) + return data['id'] + else: + log.error("Error when triggering build of %s: %s", scm_url, data) + + return None diff --git a/freshmaker/parsers/bodhi/__init__.py b/freshmaker/parsers/bodhi/__init__.py new file mode 100644 index 0000000..56f11f4 --- /dev/null +++ b/freshmaker/parsers/bodhi/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .update_complete_stable import BodhiUpdateCompleteStableParser # noqa diff --git a/freshmaker/parsers/bodhi/update_complete_stable.py b/freshmaker/parsers/bodhi/update_complete_stable.py new file mode 100644 index 0000000..b2ab173 --- /dev/null +++ b/freshmaker/parsers/bodhi/update_complete_stable.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +from freshmaker import log +from freshmaker.parsers import BaseParser +from freshmaker.events import BodhiUpdateCompleteStableEvent + + +class BodhiUpdateCompleteStableParser(BaseParser): + """Parse Bodhi message from topic bodhi.update.complete.stable""" + + name = 'BodhiUpdateCompleteStableParser' + topic_suffixes = ['bodhi.update.complete.stable'] + + def can_parse(self, topic, msg): + return any([topic.endswith(suffix) for suffix in self.topic_suffixes]) + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + msg_inner_msg = msg.get('msg') + + if not msg_inner_msg: + log.debug('Skipping message without any content with the topic "%s"', topic) + return None + + update = msg_inner_msg['update'] + return BodhiUpdateCompleteStableEvent(msg_id, + update_id=update['updateid'], + builds=update['builds'], + release=update['release']) diff --git a/freshmaker/parsers/bodhiupdate.py b/freshmaker/parsers/bodhiupdate.py deleted file mode 100644 index 646b269..0000000 --- a/freshmaker/parsers/bodhiupdate.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -from freshmaker import log -from freshmaker.parsers import BaseParser -from freshmaker.events import BodhiUpdateCompleteStable - - -class UpdateCompleteStableParser(BaseParser): - """Parse Bodhi message from topic bodhi.update.complete.stable""" - - name = 'UpdateCompleteStableParser' - topic_suffixes = ['bodhi.update.complete.stable'] - - def can_parse(self, topic, msg): - return any([topic.endswith(suffix) for suffix in self.topic_suffixes]) - - def parse(self, topic, msg): - msg_id = msg.get('msg_id') - msg_inner_msg = msg.get('msg') - - if not msg_inner_msg: - log.debug('Skipping message without any content with the topic "%s"', topic) - return None - - update = msg_inner_msg['update'] - return BodhiUpdateCompleteStable(msg_id, - update_id=update['updateid'], - builds=update['builds'], - release=update['release']) diff --git a/freshmaker/parsers/buildsys.py b/freshmaker/parsers/buildsys.py deleted file mode 100644 index b301fc2..0000000 --- a/freshmaker/parsers/buildsys.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from freshmaker import log -from freshmaker.parsers import BaseParser -from freshmaker.events import KojiTaskStateChanged - - -class BuildsysParser(BaseParser): - """ - Parser parsing task state change message from buildsys (koji), generating - KojiTaskStateChanged event. - """ - name = "BuildsysParser" - topic_suffixes = ["buildsys.task.state.change"] - - def can_parse(self, topic, msg): - log.debug(topic) - if not any([topic.endswith(s) for s in self.topic_suffixes]): - return False - return True - - def parse(self, topic, msg): - msg_id = msg.get('msg_id') - msg_inner_msg = msg.get('msg') - - # If there isn't a msg dict in msg then this message can be skipped - if not msg_inner_msg: - log.debug(('Skipping message without any content with the ' - 'topic "{0}"').format(topic)) - return None - - return KojiTaskStateChanged(msg_id, - msg_inner_msg.get('id'), - msg_inner_msg.get('new')) diff --git a/freshmaker/parsers/git/__init__.py b/freshmaker/parsers/git/__init__.py new file mode 100644 index 0000000..a57f1ad --- /dev/null +++ b/freshmaker/parsers/git/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .receive import GitReceiveParser # noqa diff --git a/freshmaker/parsers/git/receive.py b/freshmaker/parsers/git/receive.py new file mode 100644 index 0000000..7e358be --- /dev/null +++ b/freshmaker/parsers/git/receive.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from freshmaker import log +from freshmaker.parsers import BaseParser +from freshmaker.events import GitDockerfileChangeEvent +from freshmaker.events import GitModuleMetadataChangeEvent +from freshmaker.events import GitRPMSpecChangeEvent + + +class GitReceiveParser(BaseParser): + """ + Parser parsing message from dist-git. + """ + name = "GitReceiveParser" + topic_suffixes = ["git.receive"] + + def can_parse(self, topic, msg): + if not any([topic.endswith(s) for s in self.topic_suffixes]): + return False + return True + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + msg_inner_msg = msg.get('msg') + + # If there isn't a msg dict in msg then this message can be skipped + if not msg_inner_msg: + log.debug(('Skipping message without any content with the ' + 'topic "{0}"').format(topic)) + return None + + commit = msg_inner_msg.get('commit') + if not commit: + log.debug(('Skipping message without commit with the ' + 'topic "{0}"').format(topic)) + return None + + namespace = commit.get("namespace") + repo = commit.get("repo") + branch = commit.get("branch") + rev = commit.get("rev") + + log.debug(namespace) + + if namespace == "modules": + log.debug("Parsed GitModuleMetadataChangeEvent fedmsg, repo=%s, " + "branch=%s, rev=%s", repo, branch, rev) + return GitModuleMetadataChangeEvent(msg_id, repo, branch, rev) + + elif namespace == 'container': + changed_files = msg['msg']['commit']['stats']['files'] + if 'Dockerfile' in changed_files: + log.debug("Parsed GitDockerfileChangeEvent fedmsg, repo=%s, " + "branch=%s, rev=%s", repo, branch, rev) + return GitDockerfileChangeEvent(msg_id, repo, branch, rev) + + elif namespace == 'rpms': + changed_files = commit.get('stats', {}).get('files', {}).keys() + has_spec = any([i.endswith('.spec') for i in changed_files]) + if has_spec: + log.debug("Parsed GitRPMSpecChangeEvent fedmsg, repo=%s, " + "branch=%s, rev=%s", repo, branch, rev) + return GitRPMSpecChangeEvent(msg_id, repo, branch, rev) + + return None diff --git a/freshmaker/parsers/gitreceive.py b/freshmaker/parsers/gitreceive.py deleted file mode 100644 index da6811c..0000000 --- a/freshmaker/parsers/gitreceive.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - -from freshmaker import log, conf -from freshmaker.parsers import BaseParser -from freshmaker.events import ModuleMetadataUpdated -from freshmaker.events import DockerfileChanged -from freshmaker.events import RPMSpecUpdated - - -class GitReceiveParser(BaseParser): - """ - Parser parsing message from dist-git. - """ - name = "GitReceiveParser" - topic_suffixes = ["git.receive"] - - def can_parse(self, topic, msg): - if not any([topic.endswith(s) for s in self.topic_suffixes]): - return False - return True - - def parse(self, topic, msg): - msg_id = msg.get('msg_id') - msg_inner_msg = msg.get('msg') - - # If there isn't a msg dict in msg then this message can be skipped - if not msg_inner_msg: - log.debug(('Skipping message without any content with the ' - 'topic "{0}"').format(topic)) - return None - - commit = msg_inner_msg.get('commit') - if not commit: - log.debug(('Skipping message without commit with the ' - 'topic "{0}"').format(topic)) - return None - - namespace = commit.get("namespace") - rev = commit.get("rev") - repo = commit.get("repo") - branch = commit.get("branch") - - log.debug(namespace) - - if namespace == "modules": - scm_url = "%s/%s/%s.git?#%s" % (conf.git_base_url, namespace, repo, rev) - log.debug("Parsed ModuleMetadataUpdated fedmsg, scm_url=%s, " - "branch=%s", scm_url, branch) - return ModuleMetadataUpdated(msg_id, scm_url, repo, branch) - - elif namespace == 'container': - changed_files = msg['msg']['commit']['stats']['files'] - if 'Dockerfile' not in changed_files: - log.debug('Dockerfile is not changed in repo {}'.format(repo)) - return None - - log.debug('Parsed DockerfileChanged fedmsg') - repo_url = '{}/{}/{}.git'.format(conf.git_base_url, namespace, repo) - return DockerfileChanged(msg_id, repo_url=repo_url, branch=branch, - namespace=namespace, repo=repo, rev=rev) - elif namespace == 'rpms': - component = commit.get('repo') - branch = commit.get('branch') - rev = commit.get('rev') - changed_files = commit.get('stats', {}).get('files', {}).keys() - has_spec = any([i.endswith('.spec') for i in changed_files]) - if has_spec: - return RPMSpecUpdated(msg_id, component, branch, rev=rev) - - return None diff --git a/freshmaker/parsers/koji/__init__.py b/freshmaker/parsers/koji/__init__.py new file mode 100644 index 0000000..db282d1 --- /dev/null +++ b/freshmaker/parsers/koji/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .task_state_change import KojiTaskStateChangeParser # noqa diff --git a/freshmaker/parsers/koji/task_state_change.py b/freshmaker/parsers/koji/task_state_change.py new file mode 100644 index 0000000..1dc1889 --- /dev/null +++ b/freshmaker/parsers/koji/task_state_change.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from freshmaker import log +from freshmaker.parsers import BaseParser +from freshmaker.events import KojiTaskStateChangeEvent + + +class KojiTaskStateChangeParser(BaseParser): + """ + Parser parsing task state change message from buildsys (koji), generating + KojiTaskStateChanged event. + """ + name = "KojiTaskStateChangeParser" + topic_suffixes = ["buildsys.task.state.change"] + + def can_parse(self, topic, msg): + log.debug(topic) + if not any([topic.endswith(s) for s in self.topic_suffixes]): + return False + return True + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + msg_inner_msg = msg.get('msg') + + # If there isn't a msg dict in msg then this message can be skipped + if not msg_inner_msg: + log.debug(('Skipping message without any content with the ' + 'topic "{0}"').format(topic)) + return None + + return KojiTaskStateChangeEvent(msg_id, + msg_inner_msg.get('id'), + msg_inner_msg.get('new')) diff --git a/freshmaker/parsers/mbs/__init__.py b/freshmaker/parsers/mbs/__init__.py new file mode 100644 index 0000000..2a01c62 --- /dev/null +++ b/freshmaker/parsers/mbs/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .module_state_change import MBSModuleStateChangeParser # noqa diff --git a/freshmaker/parsers/mbs/module_state_change.py b/freshmaker/parsers/mbs/module_state_change.py new file mode 100644 index 0000000..a647084 --- /dev/null +++ b/freshmaker/parsers/mbs/module_state_change.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from freshmaker import log +from freshmaker.parsers import BaseParser +from freshmaker.events import MBSModuleStateChangeEvent + + +class MBSModuleStateChangeParser(BaseParser): + """ + Parser parsing message from module-build-service, generating + ModuleBuilt event. + """ + name = "MBSModuleStateChangeParser" + topic_suffixes = ["mbs.module.state.change"] + + def can_parse(self, topic, msg): + log.debug(topic) + if not any([topic.endswith(s) for s in self.topic_suffixes]): + return False + return True + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + msg_inner_msg = msg.get('msg') + + # If there isn't a msg dict in msg then this message can be skipped + if not msg_inner_msg: + log.debug(('Skipping message without any content with the ' + 'topic "{0}"').format(topic)) + return None + + return MBSModuleStateChangeEvent(msg_id, + msg_inner_msg.get('name'), + msg_inner_msg.get('stream'), + msg_inner_msg.get('id'), + msg_inner_msg.get('state')) diff --git a/freshmaker/parsers/mbsmodule.py b/freshmaker/parsers/mbsmodule.py deleted file mode 100644 index e2c4259..0000000 --- a/freshmaker/parsers/mbsmodule.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - -from freshmaker import log -from freshmaker.parsers import BaseParser -from freshmaker.events import ModuleBuilt - - -class MBSModuleParser(BaseParser): - """ - Parser parsing message from module-build-service, generating - ModuleBuilt event. - """ - name = "MBSModuleParser" - topic_suffixes = ["mbs.module.state.change"] - - def can_parse(self, topic, msg): - log.debug(topic) - if not any([topic.endswith(s) for s in self.topic_suffixes]): - return False - return True - - def parse(self, topic, msg): - msg_id = msg.get('msg_id') - msg_inner_msg = msg.get('msg') - - # If there isn't a msg dict in msg then this message can be skipped - if not msg_inner_msg: - log.debug(('Skipping message without any content with the ' - 'topic "{0}"').format(topic)) - return None - - return ModuleBuilt(msg_id, - msg_inner_msg.get('id'), - msg_inner_msg.get('state'), - msg_inner_msg.get('name'), - msg_inner_msg.get('stream')) diff --git a/freshmaker/pdc.py b/freshmaker/pdc.py index 9f90b76..dc7778c 100644 --- a/freshmaker/pdc.py +++ b/freshmaker/pdc.py @@ -27,65 +27,65 @@ from pdc_client import PDCClient import freshmaker.utils -def get_client_session(config): - """ - :param config: instance of freshmaker.config.Config - :return: pdc_client.PDCClient instance - """ - if 'ssl_verify' in inspect.getargspec(PDCClient.__init__).args: - # New API - return PDCClient( - server=config.pdc_url, - develop=config.pdc_develop, - ssl_verify=not config.pdc_insecure, - ) - else: - # Old API - return PDCClient( - server=config.pdc_url, - develop=config.pdc_develop, - insecure=config.pdc_insecure, - ) +class PDC(object): + def __init__(self, config): + # pdc_url, pdc_develop and pdc_insecure should be avaiable in config + self.config = config + self.session = self.get_client_session() + def get_client_session(self): + """ + Return pdc_client.PDCClient instance + """ + if 'ssl_verify' in inspect.getargspec(PDCClient.__init__).args: + # New API + return PDCClient( + server=self.config.pdc_url, + develop=self.config.pdc_develop, + ssl_verify=not self.config.pdc_insecure, + ) + else: + # Old API + return PDCClient( + server=self.config.pdc_url, + develop=self.config.pdc_develop, + insecure=self.config.pdc_insecure, + ) -@freshmaker.utils.retry(wait_on=(requests.ConnectTimeout, requests.ConnectionError), logger=freshmaker.log) -def get_modules(pdc_session, **kwargs): - """ - Query PDC with specified query parameters and return a list of modules. + def get_latest_modules(self, **kwargs): + """ + Query PDC with query parameters in kwargs and return a list of modules + which contains latest modules of each (module_name, module_version). - :param pdc_session: PDCClient instance - :param kwargs: query parameters in keyword arguments - :return: a list of modules - """ - modules = pdc_session['unreleasedvariants'](page_size=-1, **kwargs) - return modules + :param kwargs: query parameters in keyword arguments, should only provide + valid query parameters supported by PDC's module query API. + :return: a list of modules + """ + modules = self.get_modules(**kwargs) + active = kwargs.get('active', 'true') + latest_modules = [] + for (name, version) in set([(m.get('variant_name'), m.get('variant_version')) for m in modules]): + mods = self.get_modules(variant_name=name, variant_version=version, active=active) + latest_modules.append(sorted(mods, key=lambda x: x['variant_release']).pop()) + return list(filter(lambda x: x in latest_modules, modules)) + @freshmaker.utils.retry(wait_on=(requests.ConnectTimeout, requests.ConnectionError), logger=freshmaker.log) + def get_modules(self, **kwargs): + """ + Query PDC with specified query parameters and return a list of modules. -def get_latest_modules(pdc_session, **kwargs): - """ - Query PDC with query parameters in kwargs and return a list of modules - which contains latest modules of each (module_name, module_version). + :param kwargs: query parameters in keyword arguments + :return: a list of modules + """ + modules = self.session['unreleasedvariants'](page_size=-1, **kwargs) + return modules - :param pdc_session: PDCClient instance - :param kwargs: query parameters in keyword arguments, should only provide - valid query parameters supported by PDC's module query API. - :return: a list of modules - """ - modules = get_modules(pdc_session, **kwargs) - active = kwargs.get('active', 'true') - latest_modules = [] - for (name, version) in set([(m.get('variant_name'), m.get('variant_version')) for m in modules]): - mods = get_modules(pdc_session, variant_name=name, variant_version=version, active=active) - latest_modules.append(sorted(mods, key=lambda x: x['variant_release']).pop()) - return list(filter(lambda x: x in latest_modules, modules)) + @freshmaker.utils.retry(wait_on=(requests.ConnectTimeout, requests.ConnectionError), logger=freshmaker.log) + def find_containers_by_rpm_name(self, rpm_name): + rels = self.session['release-component-relationships'](type='ContainerIncludesRPM', + to_component_name=rpm_name) + return [rel['from_component'] for rel in rels['results']] - -@freshmaker.utils.retry(wait_on=(requests.ConnectTimeout, requests.ConnectionError), logger=freshmaker.log) -def find_containers_by_rpm_name(pdc_session, rpm_name): - rels = pdc_session['release-component-relationships'](type='ContainerIncludesRPM', - to_component_name=rpm_name) - return [rel['from_component'] for rel in rels['results']] - - -def get_release_component(pdc_session, id): - return pdc_session['release-components/{}/'.format(id)]() + @freshmaker.utils.retry(wait_on=(requests.ConnectTimeout, requests.ConnectionError), logger=freshmaker.log) + def get_release_component_by_id(self, id): + return self.session['release-components/{}/'.format(id)]() diff --git a/freshmaker/utils.py b/freshmaker/utils.py index 12b80ff..431382e 100644 --- a/freshmaker/utils.py +++ b/freshmaker/utils.py @@ -80,12 +80,26 @@ def temp_dir(logger=None, *args, **kwargs): logger.warn('Error removing %s: %s', dir, exc.strerror) -def clone_module_repo(name, dest, branch='master', user=None, logger=None): - """Clone a module repo""" - if user is None: - user = getpass.getuser() - cmd = ['git', 'clone', '-b', branch, os.path.join(conf.git_ssh_base_url % user, 'modules', name), dest] +def clone_repo(url, dest, branch='master', logger=None): + cmd = ['git', 'clone', '-b', branch, url, dest] _run_command(cmd, logger=logger) + return dest + + +def clone_distgit_repo(namespace, name, dest, branch='master', ssh=True, user=None, logger=None): + """clone a git repo""" + if ssh: + if user is None: + if hasattr(conf, 'git_user'): + user = conf.git_user + else: + user = getpass.getuser() + repo_url = conf.git_ssh_base_url % user + else: + repo_url = conf.git_base_url + + repo_url = os.path.join(repo_url, namespace, name) + return clone_repo(repo_url, dest, branch=branch, logger=logger) def add_empty_commit(repo, msg="bump", author=None, logger=None): @@ -94,20 +108,44 @@ def add_empty_commit(repo, msg="bump", author=None, logger=None): author = conf.git_author cmd = ['git', 'commit', '--allow-empty', '-m', msg, '--author={}'.format(author)] _run_command(cmd, logger=logger, rundir=repo) + return get_commit_hash(repo) -def push_repo(repo, user=None, logger=None): +def push_repo(repo, logger=None): """Push repo""" - if user is None: - user = getpass.getuser() cmd = ['git', 'push'] _run_command(cmd, logger=logger, rundir=repo) -def get_commit_hash(repo, revision='HEAD'): +def get_commit_hash(repo, branch='master', revision='HEAD', logger=None): """Get commit hash from revision""" + commit_hash = None cmd = ['git', 'rev-parse', revision] - return _run_command(cmd, rundir=repo, return_output=True).strip() + if '://' in repo: + # this is a remote repo url + with temp_dir(prefix='freshmaker-%s-' % repo.split('/').pop()) as repodir: + clone_repo(repo, repodir, branch=branch, logger=logger) + commit_hash = _run_command(cmd, rundir=repodir, return_output=True).strip() + else: + # repo is local dir + commit_hash = _run_command(cmd, rundir=repo, return_output=True).strip() + + return commit_hash + + +def bump_distgit_repo(namespace, name, branch='master', user=None, commit_author=None, commit_msg=None, logger=None): + rev = None + with temp_dir(prefix='freshmaker-%s-%s-' % (namespace, name)) as repodir: + try: + msg = commit_msg or "Bump" + clone_distgit_repo(namespace, name, repodir, branch=branch, ssh=True, user=user, logger=logger) + rev = add_empty_commit(repodir, msg=msg, author=commit_author, logger=logger) + push_repo(repodir, logger=logger) + except Exception: + if logger: + logger.error("Failed to update repo of '%s/%s:%s'.", namespace, name, branch) + return None + return rev def _run_command(command, logger=None, rundir=None, output=subprocess.PIPE, error=subprocess.PIPE, env=None, return_output=False): diff --git a/tests/handlers/test_image_builder.py b/tests/handlers/test_image_builder.py deleted file mode 100644 index 04f3394..0000000 --- a/tests/handlers/test_image_builder.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -import tempfile -import unittest - -import fedmsg.config -import pytest -import six - -from mock import patch -from mock import MagicMock -from mock import call - -from freshmaker import conf, db, models -from freshmaker.consumer import FreshmakerConsumer -from freshmaker.handlers.image_builder import DockerImageRebuildHandlerForBodhi -from tests import get_fedmsg - - -class BaseTestCase(unittest.TestCase): - def setUp(self): - db.session.remove() - db.drop_all() - db.create_all() - db.session.commit() - - def tearDown(self): - db.session.remove() - db.drop_all() - db.session.commit() - - def create_consumer(self): - hub = MagicMock() - hub.config = fedmsg.config.load_config() - return FreshmakerConsumer(hub) - - @patch("requests.request") - @patch('freshmaker.consumer.get_global_consumer') - def consume_fedmsg(self, msg, global_consumer, request): - consumer = self.create_consumer() - global_consumer.return_value = consumer - consumer.consume(msg) - - -@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') -class TestImageBuilderHandler(BaseTestCase): - - @patch('koji.read_config') - @patch('koji.ClientSession') - def test_rebuild_if_Dockerfile_changed(self, ClientSession, read_config): - read_config.return_value = { - 'server': 'https://localhost/kojihub', - 'krb_rdns': False, - 'weburl': 'https://localhost/koji', - } - - mock_session = ClientSession.return_value - mock_session.buildContainer.return_value = 123 - msg = get_fedmsg('git_receive_dockerfile_changed') - self.consume_fedmsg(msg) - - mock_session.krb_login.assert_called_once_with(proxyuser=None) - mock_session.buildContainer.assert_called_once_with( - 'git://pkgs.fedoraproject.org/container/testimage.git?#e1f39d43471fc37ec82616f76a119da4eddec787', - 'rawhide-container-candidate', - {'scratch': True, 'git_branch': 'master'}) - mock_session.logout.assert_called_once() - - events = models.Event.query.all() - self.assertEquals(len(events), 1) - self.assertEquals(events[0].message_id, msg['body']['msg_id']) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 1) - self.assertEquals(builds[0].name, 'testimage') - self.assertEquals(builds[0].type, models.ARTIFACT_TYPES['image']) - self.assertEquals(builds[0].build_id, 123) - - @patch('freshmaker.handlers.image_builder.DockerImageRebuildHandler.build_image') - def test_not_rebuild_if_Dockerfile_not_changed(self, build_image): - self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_not_changed')) - build_image.assert_not_called() - - @patch('koji.read_config') - @patch('koji.ClientSession') - def test_ensure_logout_in_whatever_case(self, ClientSession, read_config): - ClientSession.return_value.buildContainer.side_effect = RuntimeError - read_config.return_value = { - 'server': 'https://localhost/kojihub', - 'krb_rdns': False, - 'weburl': 'https://localhost/koji', - } - - self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) - - ClientSession.return_value.logout.assert_called_once() - - @patch('koji.read_config') - @patch('koji.ClientSession') - def test_ensure_do_nothing_if_fail_to_login_koji(self, ClientSession, read_config): - ClientSession.return_value.krb_login.side_effect = RuntimeError - read_config.return_value = { - 'server': 'https://localhost/kojihub', - 'krb_rdns': False, - 'weburl': 'https://localhost/koji', - } - - self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) - - ClientSession.return_value.buildContainer.assert_not_called() - - -mock_found_containers = [ - { - 'release': 'fedora-25-updates', - 'id': 5430, - 'name': 'testimage1', - 'branch': 'f25', - }, - { - 'release': 'fedora-25-updates', - 'id': 5431, - 'name': 'testimage2', - 'branch': 'f25', - }, -] - -mock_release_components = { - 5430: { - 'id': 5430, - 'release': { - 'active': True, - 'release_id': 'fedora-25-updates' - }, - 'bugzilla_component': None, - 'brew_package': None, - 'global_component': 'testimage1', - 'name': 'testimage1', - 'dist_git_branch': 'f25', - 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage1', - 'active': True, - 'type': 'container', - 'srpm': None, - }, - 5431: { - 'id': 5431, - 'release': { - 'active': True, - 'release_id': 'fedora-25-updates' - }, - 'bugzilla_component': None, - 'brew_package': None, - 'global_component': 'testimage2', - 'name': 'testimage2', - 'dist_git_branch': 'f25', - 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage2', - 'active': True, - 'type': 'container', - 'srpm': None, - } -} - - -def mock_get_release_component(pdc_session, id): - return mock_release_components[id] - - -@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') -class TestRebuildWhenBodhiUpdateStable(BaseTestCase): - - def setUp(self): - super(TestRebuildWhenBodhiUpdateStable, self).setUp() - # Use to return a temporary directory from temp_dir method. So, no need - # to delete this directory, since temp_dir ensures to do that. - self.working_dir = tempfile.mkdtemp(prefix='test-image-rebuild-') - - @patch('koji.ClientSession') - @patch('tempfile.mkdtemp') - @patch('freshmaker.handlers.image_builder._run_command') - @patch('freshmaker.handlers.image_builder.get_commit_hash') - @patch('freshmaker.handlers.image_builder.' - 'DockerImageRebuildHandlerForBodhi.get_rpms_included_in_bodhi_update') - @patch('freshmaker.handlers.image_builder.' - 'DockerImageRebuildHandlerForBodhi.get_containers_including_rpms') - @patch('freshmaker.pdc.get_release_component', new=mock_get_release_component) - def test_rebuild(self, - get_containers_including_rpms, - get_rpms_included_in_bodhi_update, - get_commit_hash, - _run_command, - mkdtemp, - ClientSession): - last_commit_hash = 'dea19c748434ec962f13d680682eee87393a4d8e' - - # A repository is not cloned actually, so just use a fake commit hash - # to construct build source URL. - get_commit_hash.return_value = last_commit_hash - - # temp_dir creates a temporary file using mkdtemp that is used for - # working directory for everything related to rebuild docker image. - # It is difficult to catch that temporary directory, so mock mkdtemp - # and use this test's own directory. - mkdtemp.return_value = self.working_dir - - get_containers_including_rpms.return_value = mock_found_containers - - session = ClientSession.return_value - session.buildContainer.side_effect = [123, 456] - - msg = get_fedmsg('bodhi_update_stable') - self.consume_fedmsg(msg) - - self.assertEqual(2, _run_command.call_count) - - _run_command.assert_has_calls([ - call(['git', 'clone', '-b', 'f25', - '{}/container/{}'.format(conf.git_base_url, 'testimage1')], - rundir=self.working_dir), - call(['git', 'clone', '-b', 'f25', - '{}/container/{}'.format(conf.git_base_url, 'testimage2')], - rundir=self.working_dir) - ]) - - self.assertEqual(2, session.krb_login.call_count) - - buildContainer = session.buildContainer - self.assertEqual(2, buildContainer.call_count) - buildContainer.assert_has_calls([ - call('{}/container/{}?#{}'.format(conf.git_base_url, - 'testimage1', - last_commit_hash), - 'f25-container-candidate', - {'scratch': True, 'git_branch': 'f25'}), - call('{}/container/{}?#{}'.format(conf.git_base_url, - 'testimage2', - last_commit_hash), - 'f25-container-candidate', - {'scratch': True, 'git_branch': 'f25'}), - ], any_order=True) - - events = models.Event.query.all() - self.assertEquals(len(events), 1) - self.assertEquals(events[0].message_id, msg['body']['msg_id']) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 2) - self.assertEquals(builds[0].name, 'testimage1') - self.assertEquals(builds[0].type, models.ARTIFACT_TYPES['image']) - self.assertEquals(builds[0].build_id, 123) - self.assertEquals(builds[1].name, 'testimage2') - self.assertEquals(builds[1].type, models.ARTIFACT_TYPES['image']) - self.assertEquals(builds[1].build_id, 456) - - -class TestContainersIncludingRPMs(unittest.TestCase): - - @patch('freshmaker.pdc.get_release_component', new=mock_get_release_component) - @patch('freshmaker.handlers.image_builder.pdc.find_containers_by_rpm_name') - def test_get_containers(self, find_containers_by_rpm_name): - expected_found_containers = [ - { - 'release': 'fedora-24-updates', - 'id': 5430, - 'name': 'testimage1', - 'branch': 'f25', - }, - { - 'release': 'fedora-24-updates', - 'id': 5431, - 'name': 'testimage2', - 'branch': 'f25', - }, - ] - find_containers_by_rpm_name.return_value = expected_found_containers - - handler = DockerImageRebuildHandlerForBodhi() - rpms = [ - {'id': 9515683, - 'name': 'community-mysql-devel', - 'nvr': 'community-mysql-devel-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18'}, - {'id': 9515682, - 'name': 'community-mysql-libs', - 'nvr': 'community-mysql-libs-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18'}, - {'id': 9515681, - 'name': 'community-mysql-server', - 'nvr': 'community-mysql-server-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18'}, - ] - - containers = handler.get_containers_including_rpms(rpms) - - self.assertEqual(3, find_containers_by_rpm_name.call_count) - found_containers = sorted(containers, key=lambda item: item['id']) - self.assertEqual(expected_found_containers, found_containers) - - -def mock_get_build_rpms(self, nvr): - """Used to patch KojiService.get_build_rpms""" - - rpms = { - 'community-mysql-5.7.18-2.fc25': [ - { - 'id': 9515683, - 'name': 'community-mysql-devel', - 'nvr': 'community-mysql-devel-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18', - }, - { - 'id': 9515682, - 'name': 'community-mysql-libs', - 'nvr': 'community-mysql-libs-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18', - }, - { - 'id': 9515681, - 'name': 'community-mysql-server', - 'nvr': 'community-mysql-server-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18', - }, - ], - 'qt5-qtwebengine-5.8.0-11.fc25': [ - { - 'id': 9571317, - 'name': 'qt5-qtwebengine-devel', - 'nvr': 'qt5-qtwebengine-devel-5.8.0-11.fc25', - 'release': '11.fc25', - 'version': '5.8.0', - }, - { - 'id': 9571316, - 'name': 'qt5-qtwebengine-examples', - 'nvr': 'qt5-qtwebengine-examples-5.8.0-11.fc25', - 'release': '11.fc25', - 'version': '5.8.0', - } - ], - } - - return rpms[nvr] - - -@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') -class TestGetRpmsIncludedInBodhiUpdate(unittest.TestCase): - """Test case for get_rpms_included_in_bodhi_update""" - - @patch('freshmaker.kojiservice.KojiService.get_build_rpms', - new=mock_get_build_rpms) - def test_get_rpms(self): - builds = [ - { - 'build_id': 884455, - 'name': 'qt5-qtwebengine', - 'nvr': 'qt5-qtwebengine-5.8.0-11.fc25', - 'release': '11.fc25', - 'version': '5.8.0', - }, - { - 'build_id': 881597, - 'name': 'community-mysql', - 'nvr': 'community-mysql-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18', - } - ] - handler = DockerImageRebuildHandlerForBodhi() - rpms = list(handler.get_rpms_included_in_bodhi_update(builds)) - - self.assertEqual(5, len(rpms)) - - rpm = filter(lambda item: item['id'] == 9515681, rpms) - self.assertEqual(1, len(rpm)) diff --git a/tests/handlers/test_mbs.py b/tests/handlers/test_mbs.py deleted file mode 100644 index 63ad838..0000000 --- a/tests/handlers/test_mbs.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import unittest -import mock -import fedmsg.config - -from mock import patch -from freshmaker.consumer import FreshmakerConsumer -from tests import get_fedmsg - - -@patch("freshmaker.consumer.get_global_consumer") -class TestMBS(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - pass - - @patch("requests.request") - def test_consumer_git_receive_module_updated(self, request, global_consumer): - """ - Tests that MBS triggers the rebuild of module as a result of GitReceive - fedmsg message. - """ - hub = mock.MagicMock() - hub.config = fedmsg.config.load_config() - consumer = FreshmakerConsumer(hub) - global_consumer.return_value = consumer - - consumer.consume(get_fedmsg("git_receive_module")) - - request.assert_called_once_with( - 'POST', 'https://mbs.fedoraproject.org/module-build-service/1/module-builds/', - headers={'Authorization': 'Bearer testingtoken'}, - json={'scmurl': u'git://pkgs.fedoraproject.org/modules/testmodule.git?#e1f39d43471fc37ec82616f76a119da4eddec787', - 'branch': u'master'}) diff --git a/tests/helpers.py b/tests/helpers.py index dd6e0cd..3cf10bf 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -78,9 +78,9 @@ class FedMsgFactory(object): } -class ModuleBuiltMessage(FedMsgFactory): +class ModuleStateChangeMessage(FedMsgFactory): def __init__(self, name, stream, state='ready', build_id=None, *args, **kwargs): - super(ModuleBuiltMessage, self).__init__(*args, **kwargs) + super(ModuleStateChangeMessage, self).__init__(*args, **kwargs) self.topic = 'org.fedoraproject.prod.mbs.module.state.change' self.name = name self.stream = stream @@ -163,9 +163,9 @@ class DistGitMessage(FedMsgFactory): self.stats['total']['lines'] += self.stats['files'][filename]['lines'] -class BuildsysTaskStateChangeMessage(FedMsgFactory): +class KojiTaskStateChangeMessage(FedMsgFactory): def __init__(self, task_id, old_state, new_state, *args, **kwargs): - super(BuildsysTaskStateChangeMessage, self).__init__(*args, **kwargs) + super(KojiTaskStateChangeMessage, self).__init__(*args, **kwargs) self.topic = 'org.fedoraproject.prod.buildsys.task.state.change' self.attribute = 'state' self.task_id = task_id diff --git a/tests/test_bodhi_update_complete_stable_handler.py b/tests/test_bodhi_update_complete_stable_handler.py new file mode 100644 index 0000000..b8751ba --- /dev/null +++ b/tests/test_bodhi_update_complete_stable_handler.py @@ -0,0 +1,275 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +import mock +import os +import pytest +import six +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa +from tests import helpers +from tests import get_fedmsg + +from freshmaker import events, db, models +from freshmaker.handlers.bodhi import BodhiUpdateCompleteStableHandler +from freshmaker.parsers.bodhi import BodhiUpdateCompleteStableParser + +mock_found_containers = [ + { + 'release': 'fedora-25-updates', + 'id': 5430, + 'name': 'testimage1', + 'branch': 'f25', + }, + { + 'release': 'fedora-25-updates', + 'id': 5431, + 'name': 'testimage2', + 'branch': 'f25', + }, +] + +mock_release_components = { + 5430: { + 'id': 5430, + 'release': { + 'active': True, + 'release_id': 'fedora-25-updates' + }, + 'bugzilla_component': None, + 'brew_package': None, + 'global_component': 'testimage1', + 'name': 'testimage1', + 'dist_git_branch': 'f25', + 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage1', + 'active': True, + 'type': 'container', + 'srpm': None, + }, + 5431: { + 'id': 5431, + 'release': { + 'active': True, + 'release_id': 'fedora-25-updates' + }, + 'bugzilla_component': None, + 'brew_package': None, + 'global_component': 'testimage2', + 'name': 'testimage2', + 'dist_git_branch': 'f25', + 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage2', + 'active': True, + 'type': 'container', + 'srpm': None, + } +} + + +def mock_get_release_component_by_id(id): + return mock_release_components[id] + + +@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') +class BodhiUpdateCompleteStableHandlerTest(helpers.FreshmakerTestCase): + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + events.BaseEvent.register_parser(BodhiUpdateCompleteStableParser) + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + @mock.patch('freshmaker.handlers.bodhi.update_complete_stable.PDC') + @mock.patch('freshmaker.handlers.bodhi.update_complete_stable.utils') + @mock.patch('freshmaker.handlers.bodhi.update_complete_stable.conf') + def test_trigger_rebuild_container_when_receives_bodhi_update_complete_stable_message(self, conf, utils, PDC): + + conf.git_base_url = 'git://pkgs.fedoraproject.org' + + handler = BodhiUpdateCompleteStableHandler() + handler.get_rpms_included_in_bodhi_update = mock.Mock() + + handler.get_containers_including_rpms = mock.Mock() + + containers = [ + { + 'release': 'fedora-25-updates', + 'id': 5430, + 'name': 'testimage1', + 'branch': 'f25', + }, + { + 'release': 'fedora-25-updates', + 'id': 5431, + 'name': 'testimage2', + 'branch': 'f25', + }, + ] + handler.get_containers_including_rpms.return_value = containers + + utils.get_commit_hash.side_effect = ['c123', 'c456'] + + handler.build_container = mock.Mock() + handler.build_container.side_effect = [123, 456] + + msg = get_fedmsg('bodhi_update_stable') + event = self.get_event_from_msg(msg) + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + self.assertEqual(handler.build_container.call_args_list, + [mock.call('testimage1', 'f25', 'c123'), + mock.call('testimage2', 'f25', 'c456')]) + + events = models.Event.query.all() + self.assertEqual(len(events), 1) + self.assertEqual(events[0].message_id, msg['body']['msg_id']) + builds = models.ArtifactBuild.query.all() + self.assertEqual(len(builds), 2) + self.assertEqual(builds[0].name, 'testimage1') + self.assertEqual(builds[0].type, models.ARTIFACT_TYPES['image']) + self.assertEqual(builds[0].build_id, 123) + self.assertEqual(builds[1].name, 'testimage2') + self.assertEqual(builds[1].type, models.ARTIFACT_TYPES['image']) + self.assertEqual(builds[1].build_id, 456) + + @mock.patch('freshmaker.handlers.bodhi.update_complete_stable.PDC') + @mock.patch('freshmaker.handlers.bodhi.update_complete_stable.utils') + @mock.patch('freshmaker.handlers.bodhi.update_complete_stable.conf') + def test_get_containers_including_rpms(self, conf, utils, PDC): + expected_found_containers = [ + { + 'release': 'fedora-24-updates', + 'id': 5430, + 'name': 'testimage1', + 'branch': 'f25', + }, + { + 'release': 'fedora-24-updates', + 'id': 5431, + 'name': 'testimage2', + 'branch': 'f25', + }, + ] + pdc = PDC(conf) + pdc.find_containers_by_rpm_name.return_value = expected_found_containers + + handler = BodhiUpdateCompleteStableHandler() + rpms = [ + {'id': 9515683, + 'name': 'community-mysql-devel', + 'nvr': 'community-mysql-devel-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18'}, + {'id': 9515682, + 'name': 'community-mysql-libs', + 'nvr': 'community-mysql-libs-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18'}, + {'id': 9515681, + 'name': 'community-mysql-server', + 'nvr': 'community-mysql-server-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18'}, + ] + + containers = handler.get_containers_including_rpms(rpms) + + self.assertEqual(3, pdc.find_containers_by_rpm_name.call_count) + found_containers = sorted(containers, key=lambda item: item['id']) + self.assertEqual(expected_found_containers, found_containers) + + @mock.patch('freshmaker.kojiservice.KojiService.get_build_rpms') + def test_get_rpms_included_in_bohdhi_update(self, get_build_rpms): + rpms = { + 'community-mysql-5.7.18-2.fc25': [ + { + 'id': 9515683, + 'name': 'community-mysql-devel', + 'nvr': 'community-mysql-devel-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + }, + { + 'id': 9515682, + 'name': 'community-mysql-libs', + 'nvr': 'community-mysql-libs-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + }, + { + 'id': 9515681, + 'name': 'community-mysql-server', + 'nvr': 'community-mysql-server-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + }, + ], + 'qt5-qtwebengine-5.8.0-11.fc25': [ + { + 'id': 9571317, + 'name': 'qt5-qtwebengine-devel', + 'nvr': 'qt5-qtwebengine-devel-5.8.0-11.fc25', + 'release': '11.fc25', + 'version': '5.8.0', + }, + { + 'id': 9571316, + 'name': 'qt5-qtwebengine-examples', + 'nvr': 'qt5-qtwebengine-examples-5.8.0-11.fc25', + 'release': '11.fc25', + 'version': '5.8.0', + } + ], + } + + get_build_rpms.side_effect = lambda x: rpms[x] + + builds = [ + { + 'build_id': 884455, + 'name': 'qt5-qtwebengine', + 'nvr': 'qt5-qtwebengine-5.8.0-11.fc25', + 'release': '11.fc25', + 'version': '5.8.0', + }, + { + 'build_id': 881597, + 'name': 'community-mysql', + 'nvr': 'community-mysql-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + } + ] + handler = BodhiUpdateCompleteStableHandler() + rpms = list(handler.get_rpms_included_in_bodhi_update(builds)) + + self.assertEqual(5, len(rpms)) + + rpm = filter(lambda item: item['id'] == 9515681, rpms) + self.assertEqual(1, len(rpm)) diff --git a/tests/test_buildsys_handler.py b/tests/test_buildsys_handler.py deleted file mode 100644 index 6c0e181..0000000 --- a/tests/test_buildsys_handler.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import sys -import unittest - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa -from tests import helpers - -from freshmaker import events, db, models -from freshmaker.handlers.buildsys import BuildsysHandler -from freshmaker.parsers.buildsys import BuildsysParser - - -class BuildsysHandlerTest(helpers.FreshmakerTestCase): - def setUp(self): - db.session.remove() - db.drop_all() - db.create_all() - db.session.commit() - - events.BaseEvent.register_parser(BuildsysParser) - - def tearDown(self): - db.session.remove() - db.drop_all() - db.session.commit() - - def test_can_handle_koji_task_state_changed_event(self): - """ - Tests buildsys handler can handle koji task state changed message - """ - m = helpers.BuildsysTaskStateChangeMessage(123, 'OPEN', 'FAILED') - msg = m.produce() - event = self.get_event_from_msg(msg) - handler = BuildsysHandler() - self.assertTrue(handler.can_handle(event)) - - def test_update_build_state_on_koji_task_state_changed_event(self): - """ - Tests build state will be updated when receives koji task state changed message - """ - task_id = 123 - ev = models.Event.create(db.session, 'test_msg_id') - build = models.ArtifactBuild.create(db.session, - ev, - 'testimage', - models.ARTIFACT_TYPES['image'], - task_id) - db.session.add(ev) - db.session.add(build) - db.session.commit() - - m = helpers.BuildsysTaskStateChangeMessage(task_id, 'OPEN', 'FAILED') - msg = m.produce() - event = self.get_event_from_msg(msg) - - handler = BuildsysHandler() - handler.handle(event) - build = models.ArtifactBuild.query.all()[0] - self.assertEqual(build.state, models.BUILD_STATES['failed']) - - m = helpers.BuildsysTaskStateChangeMessage(task_id, 'OPEN', 'CLOSED') - msg = m.produce() - event = self.get_event_from_msg(msg) - - handler = BuildsysHandler() - handler.handle(event) - build = models.ArtifactBuild.query.all()[0] - self.assertEqual(build.state, models.BUILD_STATES['done']) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_consumer.py b/tests/test_consumer.py index 353402c..8087d55 100644 --- a/tests/test_consumer.py +++ b/tests/test_consumer.py @@ -32,9 +32,9 @@ class ConsumerTest(unittest.TestCase): def tearDown(self): pass - @mock.patch("freshmaker.handlers.mbs.MBS.handle_module_built") + @mock.patch("freshmaker.handlers.mbs.module_state_change.MBSModuleStateChangeHandler.handle") @mock.patch("freshmaker.consumer.get_global_consumer") - def test_consumer_processing_message(self, global_consumer, handle_module_built): + def test_consumer_processing_message(self, global_consumer, handle): """ Tests that consumer parses the message, forwards the event to proper handler and is able to get the further work from @@ -57,7 +57,7 @@ class ConsumerTest(unittest.TestCase): } }} - handle_module_built.return_value = [freshmaker.events.TestingEvent("ModuleBuilt handled")] + handle.return_value = [freshmaker.events.TestingEvent("ModuleBuilt handled")] consumer.consume(msg) event = consumer.incoming.get() diff --git a/tests/test_git_dockerfile_change_handler.py b/tests/test_git_dockerfile_change_handler.py new file mode 100644 index 0000000..915d325 --- /dev/null +++ b/tests/test_git_dockerfile_change_handler.py @@ -0,0 +1,126 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +import unittest + +import fedmsg.config +import pytest +import six + +from mock import patch +from mock import MagicMock + +from freshmaker import db, models +from freshmaker.consumer import FreshmakerConsumer +from tests import get_fedmsg + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def create_consumer(self): + hub = MagicMock() + hub.config = fedmsg.config.load_config() + return FreshmakerConsumer(hub) + + @patch("requests.request") + @patch('freshmaker.consumer.get_global_consumer') + def consume_fedmsg(self, msg, global_consumer, request): + consumer = self.create_consumer() + global_consumer.return_value = consumer + consumer.consume(msg) + + +@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') +class GitDockerfileChangeHandlerTest(BaseTestCase): + + @patch('koji.read_config') + @patch('koji.ClientSession') + def test_rebuild_if_dockerfile_changed(self, ClientSession, read_config): + read_config.return_value = { + 'server': 'https://localhost/kojihub', + 'krb_rdns': False, + 'weburl': 'https://localhost/koji', + } + + mock_session = ClientSession.return_value + mock_session.buildContainer.return_value = 123 + msg = get_fedmsg('git_receive_dockerfile_changed') + self.consume_fedmsg(msg) + + mock_session.krb_login.assert_called_once_with(proxyuser=None) + mock_session.buildContainer.assert_called_once_with( + 'git://pkgs.fedoraproject.org/container/testimage.git?#e1f39d43471fc37ec82616f76a119da4eddec787', + 'rawhide-container-candidate', + {'scratch': True, 'git_branch': 'master'}) + mock_session.logout.assert_called_once() + + events = models.Event.query.all() + self.assertEqual(len(events), 1) + self.assertEqual(events[0].message_id, msg['body']['msg_id']) + builds = models.ArtifactBuild.query.all() + self.assertEqual(len(builds), 1) + self.assertEqual(builds[0].name, 'testimage') + self.assertEqual(builds[0].type, models.ARTIFACT_TYPES['image']) + self.assertEqual(builds[0].build_id, 123) + + @patch('freshmaker.handlers.git.dockerfile_change.GitDockerfileChangeHandler.build_container') + def test_not_rebuild_if_dockerfile_not_changed(self, build_container): + self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_not_changed')) + build_container.assert_not_called() + + @patch('koji.read_config') + @patch('koji.ClientSession') + def test_ensure_logout_in_whatever_case(self, ClientSession, read_config): + ClientSession.return_value.buildContainer.side_effect = RuntimeError + read_config.return_value = { + 'server': 'https://localhost/kojihub', + 'krb_rdns': False, + 'weburl': 'https://localhost/koji', + } + + self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) + + ClientSession.return_value.logout.assert_called_once() + + @patch('koji.read_config') + @patch('koji.ClientSession') + def test_ensure_do_nothing_if_fail_to_login_koji(self, ClientSession, read_config): + ClientSession.return_value.krb_login.side_effect = RuntimeError + read_config.return_value = { + 'server': 'https://localhost/kojihub', + 'krb_rdns': False, + 'weburl': 'https://localhost/koji', + } + + self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) + + ClientSession.return_value.buildContainer.assert_not_called() diff --git a/tests/test_git_module_metadata_change_handler.py b/tests/test_git_module_metadata_change_handler.py new file mode 100644 index 0000000..a02903c --- /dev/null +++ b/tests/test_git_module_metadata_change_handler.py @@ -0,0 +1,92 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import sys +import unittest +import mock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa +from tests import helpers + +from freshmaker import events, db, models +from freshmaker.handlers.git import GitModuleMetadataChangeHandler +from freshmaker.parsers.git import GitReceiveParser + + +class GitModuleMetadataChangeHandlerTest(helpers.FreshmakerTestCase): + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + events.BaseEvent.register_parser(GitReceiveParser) + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def test_can_handle_module_metadata_change_event(self): + """ + Tests handler can handle module metadata change message + """ + m = helpers.DistGitMessage('modules', 'testmodule', 'master', '123') + m.add_changed_file('testmodule.yaml', 1, 1) + msg = m.produce() + + event = self.get_event_from_msg(msg) + + handler = GitModuleMetadataChangeHandler() + self.assertTrue(handler.can_handle(event)) + + def test_can_rebuild_module_when_module_metadata_changed(self): + """ + Tests handler can rebuild module when module metadata is changed in dist-git + """ + m = helpers.DistGitMessage('modules', 'testmodule', 'master', '12345') + m.add_changed_file('testmodule.yaml', 1, 1) + msg = m.produce() + + event = self.get_event_from_msg(msg) + + handler = GitModuleMetadataChangeHandler() + handler.build_module = mock.Mock() + handler.build_module.return_value = 123 + + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + self.assertEqual(handler.build_module.call_args_list, + [mock.call('testmodule', 'master', '12345')]) + + event_list = models.Event.query.all() + self.assertEqual(len(event_list), 1) + self.assertEqual(event_list[0].message_id, event.msg_id) + builds = models.ArtifactBuild.query.all() + self.assertEqual(len(builds), 1) + self.assertEqual(builds[0].name, 'testmodule') + self.assertEqual(builds[0].type, models.ARTIFACT_TYPES['module']) + self.assertEqual(builds[0].build_id, 123) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_git_rpm_spec_change_handler.py b/tests/test_git_rpm_spec_change_handler.py new file mode 100644 index 0000000..1cae34f --- /dev/null +++ b/tests/test_git_rpm_spec_change_handler.py @@ -0,0 +1,119 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import sys +import unittest +import mock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa +from tests import helpers + +from freshmaker import events, db, models +from freshmaker.handlers.git import GitRPMSpecChangeHandler +from freshmaker.parsers.git import GitReceiveParser + + +class GitRPMSpecChangeHandlerTest(helpers.FreshmakerTestCase): + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + events.BaseEvent.register_parser(GitReceiveParser) + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def test_can_handle_dist_git_message_with_rpm_spec_changed(self): + """ + Tests handler can handle rpm spec change event + """ + m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') + m.add_changed_file('bash.spec', 1, 1) + msg = m.produce() + + event = self.get_event_from_msg(msg) + + handler = GitRPMSpecChangeHandler() + self.assertTrue(handler.can_handle(event)) + + def test_can_not_handle_dist_git_message_without_rpm_spec_changed(self): + """ + Tests can not handle dist git message that spec file is not changed. + """ + + m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') + m.add_changed_file('test.c', 1, 1) + msg = m.produce() + + event = self.get_event_from_msg(msg) + + handler = GitRPMSpecChangeHandler() + self.assertFalse(handler.can_handle(event)) + + @mock.patch('freshmaker.handlers.git.rpm_spec_change.PDC') + @mock.patch('freshmaker.handlers.git.rpm_spec_change.utils') + @mock.patch('freshmaker.handlers.git.rpm_spec_change.conf') + def test_can_rebuild_modules_has_rpm_included(self, conf, utils, PDC): + """ + Test handler can rebuild modules which include the rpm. + """ + conf.git_base_url = "git://pkgs.fedoraproject.org" + + m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') + m.add_changed_file('bash.spec', 1, 1) + msg = m.produce() + + event = self.get_event_from_msg(msg) + + mod_info = helpers.PDCModuleInfo('testmodule', 'master', '20170412010101') + mod_info.add_rpm("bash-1.2.3-4.f26.rpm") + mod = mod_info.produce() + pdc = PDC.return_value + pdc.get_latest_modules.return_value = [mod] + + commitid = '9287eb8eb4c4c60f73b4a59f228a673846d940c6' + utils.bump_distgit_repo.return_value = commitid + + handler = GitRPMSpecChangeHandler() + handler.build_module = mock.Mock() + handler.build_module.return_value = 123 + + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + handler.build_module.assert_called_with('testmodule', 'master', commitid) + + event_list = models.Event.query.all() + self.assertEqual(len(event_list), 1) + self.assertEqual(event_list[0].message_id, event.msg_id) + builds = models.ArtifactBuild.query.all() + self.assertEqual(len(builds), 1) + self.assertEqual(builds[0].name, 'testmodule') + self.assertEqual(builds[0].type, models.ARTIFACT_TYPES['module']) + self.assertEqual(builds[0].build_id, 123) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_koji_task_state_change_handler.py b/tests/test_koji_task_state_change_handler.py new file mode 100644 index 0000000..2fcc0c7 --- /dev/null +++ b/tests/test_koji_task_state_change_handler.py @@ -0,0 +1,96 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa +from tests import helpers + +from freshmaker import events, db, models +from freshmaker.handlers.koji import KojiTaskStateChangeHandler +from freshmaker.parsers.koji import KojiTaskStateChangeParser + + +class KojiTaskStateChangeHandlerTest(helpers.FreshmakerTestCase): + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + events.BaseEvent.register_parser(KojiTaskStateChangeParser) + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def test_can_handle_koji_task_state_change_message(self): + """ + Tests buildsys handler can handle koji task state changed message + """ + m = helpers.KojiTaskStateChangeMessage(123, 'OPEN', 'FAILED') + msg = m.produce() + event = self.get_event_from_msg(msg) + handler = KojiTaskStateChangeHandler() + self.assertTrue(handler.can_handle(event)) + + def test_update_build_state_on_koji_task_state_change_event(self): + """ + Tests build state will be updated when receives koji task state changed message + """ + task_id = 123 + ev = models.Event.create(db.session, 'test_msg_id') + build = models.ArtifactBuild.create(db.session, + ev, + 'testimage', + models.ARTIFACT_TYPES['image'], + task_id) + db.session.add(ev) + db.session.add(build) + db.session.commit() + + m = helpers.KojiTaskStateChangeMessage(task_id, 'OPEN', 'FAILED') + msg = m.produce() + event = self.get_event_from_msg(msg) + + handler = KojiTaskStateChangeHandler() + + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + build = models.ArtifactBuild.query.all()[0] + self.assertEqual(build.state, models.BUILD_STATES['failed']) + + m = helpers.KojiTaskStateChangeMessage(task_id, 'OPEN', 'CLOSED') + msg = m.produce() + event = self.get_event_from_msg(msg) + + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + build = models.ArtifactBuild.query.all()[0] + self.assertEqual(build.state, models.BUILD_STATES['done']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index b60bc67..4983264 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -287,7 +287,6 @@ class TestEntityVersion(unittest.TestCase): cert=self.fake_cert_file, private_key=self.fake_private_key, entity_versions=self.fake_entity_versions) - fake_request = {} lb.find_container_repositories({}) _make_request.assert_called_once_with('find/containerRepository/', {}) @@ -301,7 +300,6 @@ class TestEntityVersion(unittest.TestCase): cert=self.fake_cert_file, private_key=self.fake_private_key, entity_versions=self.fake_entity_versions) - fake_request = {} lb.find_container_images({}) _make_request.assert_called_once_with('find/containerImage/0.0.11', {}) @@ -319,7 +317,6 @@ class TestEntityVersion(unittest.TestCase): lb = LightBlue(server_url=self.fake_server_url, cert=self.fake_cert_file, private_key=self.fake_private_key) - fake_request = {} lb.find_container_repositories({}) lb.find_container_images({}) diff --git a/tests/test_mbs_handler.py b/tests/test_mbs_handler.py deleted file mode 100644 index 6fc0a61..0000000 --- a/tests/test_mbs_handler.py +++ /dev/null @@ -1,378 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import sys -import unittest -import mock - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa -from tests import helpers - -from freshmaker import events, db, models -from freshmaker.handlers.mbs import MBS -from freshmaker.parsers.mbsmodule import MBSModuleParser -from freshmaker.parsers.gitreceive import GitReceiveParser - - -class MBSHandlerTest(helpers.FreshmakerTestCase): - def setUp(self): - db.session.remove() - db.drop_all() - db.create_all() - db.session.commit() - - events.BaseEvent.register_parser(MBSModuleParser) - events.BaseEvent.register_parser(GitReceiveParser) - - def tearDown(self): - db.session.remove() - db.drop_all() - db.session.commit() - - def test_can_handle_module_built_event(self): - """ - Tests MBS handler can handle module built message - """ - for state in ['init', 'wait', 'build', 'done', 'failed', 'ready']: - msg = helpers.ModuleBuiltMessage('testmodule', 'master', state=state).produce() - event = self.get_event_from_msg(msg) - - handler = MBS() - self.assertTrue(handler.can_handle(event)) - - @mock.patch('freshmaker.pdc.get_modules') - @mock.patch('freshmaker.handlers.mbs.utils') - @mock.patch('freshmaker.handlers.mbs.conf') - def test_rebuild_depending_modules_on_module_built_event(self, conf, utils, get_modules): - """ - Tests MBS handler can rebuild all modules which depend on the module - in module built event. - """ - msg = helpers.ModuleBuiltMessage('testmodule', 'master', state='ready').produce() - event = self.get_event_from_msg(msg) - - handler = MBS() - - mod2_r1_info = helpers.PDCModuleInfo('testmodule2', 'master', '20170412010101') - mod2_r1_info.add_build_dep('testmodule', 'master') - mod2_r1 = mod2_r1_info.produce() - - mod3_r1_info = helpers.PDCModuleInfo('testmodule3', 'master', '20170412010201') - mod3_r1_info.add_build_dep('testmodule', 'master') - mod3_r1 = mod3_r1_info.produce() - - def mock_get_modules(pdc_session, **kwargs): - name = kwargs.get('variant_name', None) - version = kwargs.get('variant_version', None) - - if name == 'testmodule2' and version == 'master': - return [mod2_r1] - elif name == 'testmodule3' and version == 'master': - return [mod3_r1] - else: - return [mod2_r1, mod3_r1] - - get_modules.side_effect = mock_get_modules - conf.git_base_url = "git://pkgs.fedoraproject.org" - utils.get_commit_hash.side_effect = [ - "fae7848fa47a854f25b782aa64441040a6d86544", - "43ec03000d249231bc7135b11b810afc96e90efb", - ] - handler.rebuild_module = mock.Mock() - handler.rebuild_module.side_effect = [123, 456] - handler.handle_module_built(event) - - self.assertEqual(handler.rebuild_module.call_args_list, - [mock.call(u'git://pkgs.fedoraproject.org/modules/testmodule2.git?#fae7848fa47a854f25b782aa64441040a6d86544', u'master'), - mock.call(u'git://pkgs.fedoraproject.org/modules/testmodule3.git?#43ec03000d249231bc7135b11b810afc96e90efb', u'master')]) - - event_list = models.Event.query.all() - self.assertEquals(len(event_list), 1) - self.assertEquals(event_list[0].message_id, event.msg_id) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 2) - self.assertEquals(builds[0].name, mod2_r1['variant_name']) - self.assertEquals(builds[0].type, models.ARTIFACT_TYPES['module']) - self.assertEquals(builds[0].build_id, 123) - self.assertEquals(builds[1].name, mod3_r1['variant_name']) - self.assertEquals(builds[1].build_id, 456) - self.assertEquals(builds[1].type, models.ARTIFACT_TYPES['module']) - - @mock.patch('freshmaker.pdc.get_modules') - @mock.patch('freshmaker.handlers.mbs.utils') - @mock.patch('freshmaker.handlers.mbs.conf') - def test_only_rebuild_latest_depending_modules_on_module_built_event(self, conf, utils, get_modules): - """ - Tests MBS handler only rebuild latest depending modules. If there is a - module only has old release depends on the module, it won't be rebuilt. - """ - msg = helpers.ModuleBuiltMessage('testmodule', 'master', state='ready').produce() - event = self.get_event_from_msg(msg) - - handler = MBS() - - mod2_r1_info = helpers.PDCModuleInfo('testmodule2', 'master', '20170412010101') - mod2_r1_info.add_build_dep('testmodule', 'master') - mod2_r1 = mod2_r1_info.produce() - - mod3_r1_info = helpers.PDCModuleInfo('testmodule3', 'master', '20170412010101') - mod3_r1_info.add_build_dep('testmodule', 'master') - mod3_r1 = mod3_r1_info.produce() - - mod3_r2_info = helpers.PDCModuleInfo('testmodule3', 'master', '20170412010201') - mod3_r2_info.add_build_dep('testmodule', 'master') - mod3_r2 = mod3_r2_info.produce() - - def mock_get_modules(pdc_session, **kwargs): - name = kwargs.get('variant_name', None) - version = kwargs.get('variant_version', None) - - if name == 'testmodule2' and version == 'master': - return [mod2_r1] - elif name == 'testmodule3' and version == 'master': - return [mod3_r1, mod3_r2] - else: - return [mod2_r1, mod3_r1] - - # query for testmodule3 releases, get mod3_r1 and mod3_r2, - # only mod3_r1 depends on testmodule, and r1 < r2. - get_modules.side_effect = mock_get_modules - conf.git_base_url = "git://pkgs.fedoraproject.org" - utils.get_commit_hash.side_effect = [ - "fae7848fa47a854f25b782aa64441040a6d86544", - "43ec03000d249231bc7135b11b810afc96e90efb", - ] - handler.rebuild_module = mock.Mock() - handler.rebuild_module.return_value = 123 - handler.handle_module_built(event) - - self.assertEqual(handler.rebuild_module.call_args_list, - [mock.call(u'git://pkgs.fedoraproject.org/modules/testmodule2.git?#fae7848fa47a854f25b782aa64441040a6d86544', u'master')]) - - event_list = models.Event.query.all() - self.assertEquals(len(event_list), 1) - self.assertEquals(event_list[0].message_id, event.msg_id) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 1) - self.assertEquals(builds[0].name, mod2_r1['variant_name']) - self.assertEquals(builds[0].type, models.ARTIFACT_TYPES['module']) - self.assertEquals(builds[0].build_id, 123) - - def test_can_handle_rpm_spec_updated_event(self): - """ - Tests MBS handler can handle rpm spec updated event - """ - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('bash.spec', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - handler = MBS() - self.assertTrue(handler.can_handle(event)) - - def test_can_not_handle_no_spec_updated_dist_git_event(self): - """ - Tests RPMSpechandler can not handle dist git message that - spec file is not updated. - """ - - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('test.c', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - handler = MBS() - self.assertFalse(handler.can_handle(event)) - - @mock.patch('freshmaker.handlers.mbs.utils') - @mock.patch('freshmaker.handlers.mbs.pdc') - @mock.patch('freshmaker.handlers.mbs.conf') - def test_trigger_module_rebuild_when_rpm_spec_updated(self, conf, pdc, utils): - """ - Test RPMSpecHandler can trigger module rebuild when spec - file of rpm in module updated. - """ - conf.git_base_url = "git://pkgs.fedoraproject.org" - - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('bash.spec', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - mod_info = helpers.PDCModuleInfo('testmodule', 'master', '20170412010101') - mod_info.add_rpm("bash-1.2.3-4.f26.rpm") - mod = mod_info.produce() - pdc.get_latest_modules.return_value = [mod] - - commitid = '9287eb8eb4c4c60f73b4a59f228a673846d940c6' - utils.get_commit_hash.return_value = commitid - - handler = MBS() - handler.rebuild_module = mock.Mock() - handler.rebuild_module.return_value = 123 - handler.handle(event) - self.assertEqual(handler.rebuild_module.call_args_list, - [mock.call('git://pkgs.fedoraproject.org/modules/testmodule.git?#%s' % commitid, 'master')]) - - event_list = models.Event.query.all() - self.assertEquals(len(event_list), 1) - self.assertEquals(event_list[0].message_id, event.msg_id) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 1) - self.assertEquals(builds[0].name, 'testmodule') - self.assertEquals(builds[0].type, models.ARTIFACT_TYPES['module']) - self.assertEquals(builds[0].build_id, 123) - - @mock.patch('freshmaker.handlers.mbs.utils') - @mock.patch('freshmaker.handlers.mbs.pdc') - @mock.patch('freshmaker.handlers.mbs.conf') - def test_update_build_state_in_db(self, conf, pdc, utils): - """ - Test build state in db will be updated when receives module build - state change message. - """ - - # trigger a build on rpm spec updated event first - conf.git_base_url = "git://pkgs.fedoraproject.org" - - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('bash.spec', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - mod_info = helpers.PDCModuleInfo('testmodule', 'master', '20170412010101') - mod_info.add_rpm("bash-1.2.3-4.f26.rpm") - mod = mod_info.produce() - pdc.get_latest_modules.return_value = [mod] - - commitid = '9287eb8eb4c4c60f73b4a59f228a673846d940c6' - utils.get_commit_hash.return_value = commitid - - handler = MBS() - handler.rebuild_module = mock.Mock() - handler.rebuild_module.return_value = 123 - handler.handle(event) - self.assertEqual(handler.rebuild_module.call_args_list, - [mock.call('git://pkgs.fedoraproject.org/modules/testmodule.git?#%s' % commitid, 'master')]) - - event_list = models.Event.query.all() - self.assertEquals(len(event_list), 1) - self.assertEquals(event_list[0].message_id, event.msg_id) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 1) - self.assertEquals(builds[0].name, 'testmodule') - self.assertEquals(builds[0].type, models.ARTIFACT_TYPES['module']) - self.assertEquals(builds[0].build_id, 123) - self.assertEquals(builds[0].state, models.BUILD_STATES['build']) - - # update build state when receive module built messages - # build is failed - msg = helpers.ModuleBuiltMessage('testmodule', 'master', state='failed', build_id=123).produce() - event = self.get_event_from_msg(msg) - handler.handle(event) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 1) - # build state updated to 'failed' - self.assertEquals(builds[0].state, models.BUILD_STATES['failed']) - - # build is ready - pdc.get_latest_modules.return_value = [] - msg = helpers.ModuleBuiltMessage('testmodule', 'master', state='ready', build_id=123).produce() - event = self.get_event_from_msg(msg) - handler.handle(event) - builds = models.ArtifactBuild.query.all() - self.assertEquals(len(builds), 1) - # build state updated to 'done' - self.assertEquals(builds[0].state, models.BUILD_STATES['done']) - - @mock.patch('freshmaker.handlers.mbs.utils') - @mock.patch('freshmaker.handlers.mbs.pdc') - @mock.patch('freshmaker.handlers.conf') - def test_module_is_not_allowed_to_be_built_in_whitelist(self, conf, pdc, utils): - conf.handler_build_whitelist = { - "MBS": { - "RPMSpecUpdated": { - "module": [ - { - 'name': 'test-.*', - }, - ], - }, - }, - } - conf.handler_build_blacklist = {} - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('bash.spec', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - mod_info = helpers.PDCModuleInfo('testmodule', 'master', '20170412010101') - mod_info.add_rpm("bash-1.2.3-4.f26.rpm") - mod = mod_info.produce() - pdc.get_latest_modules.return_value = [mod] - event = self.get_event_from_msg(msg) - handler = MBS() - handler.rebuild_module = mock.Mock() - handler.rebuild_module.return_value = None - handler.handle(event) - handler.rebuild_module.assert_not_called() - - @mock.patch('freshmaker.handlers.mbs.utils') - @mock.patch('freshmaker.handlers.mbs.pdc') - @mock.patch('freshmaker.handlers.conf') - def test_module_is_not_allowed_to_be_built_in_blacklist(self, conf, pdc, utils): - conf.handler_build_whitelist = {} - conf.handler_build_blacklist = { - "MBS": { - "RPMSpecUpdated": { - "module": [ - { - 'name': 'testmodule', - }, - ], - }, - }, - } - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('bash.spec', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - mod_info = helpers.PDCModuleInfo('testmodule', 'master', '20170412010101') - mod_info.add_rpm("bash-1.2.3-4.f26.rpm") - mod = mod_info.produce() - pdc.get_latest_modules.return_value = [mod] - event = self.get_event_from_msg(msg) - handler = MBS() - handler.rebuild_module = mock.Mock() - handler.rebuild_module.return_value = None - handler.handle(event) - handler.rebuild_module.assert_not_called() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_mbs_module_state_change_handler.py b/tests/test_mbs_module_state_change_handler.py new file mode 100644 index 0000000..95a3a55 --- /dev/null +++ b/tests/test_mbs_module_state_change_handler.py @@ -0,0 +1,183 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import sys +import unittest +import mock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa +from tests import helpers + +from freshmaker import events, db, models +from freshmaker.handlers.mbs import MBSModuleStateChangeHandler +from freshmaker.parsers.mbs import MBSModuleStateChangeParser + + +class MBSModuleStateChangeHandlerTest(helpers.FreshmakerTestCase): + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + events.BaseEvent.register_parser(MBSModuleStateChangeParser) + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def test_can_handle_module_state_change_event(self): + """ + Tests MBS handler can handle module built message + """ + for state in ['init', 'wait', 'build', 'done', 'failed', 'ready']: + msg = helpers.ModuleStateChangeMessage('testmodule', 'master', state=state).produce() + event = self.get_event_from_msg(msg) + + handler = MBSModuleStateChangeHandler() + self.assertTrue(handler.can_handle(event)) + + @mock.patch('freshmaker.handlers.mbs.module_state_change.PDC') + @mock.patch('freshmaker.handlers.mbs.module_state_change.utils') + @mock.patch('freshmaker.handlers.mbs.module_state_change.conf') + def test_can_rebuild_depending_modules(self, conf, utils, PDC): + """ + Tests handler can rebuild all modules which depend on the module + in module state change event. + """ + msg = helpers.ModuleStateChangeMessage('testmodule', 'master', state='ready').produce() + event = self.get_event_from_msg(msg) + + mod2_r1_info = helpers.PDCModuleInfo('testmodule2', 'master', '20170412010101') + mod2_r1_info.add_build_dep('testmodule', 'master') + mod2_r1 = mod2_r1_info.produce() + + mod3_r1_info = helpers.PDCModuleInfo('testmodule3', 'master', '20170412010201') + mod3_r1_info.add_build_dep('testmodule', 'master') + mod3_r1 = mod3_r1_info.produce() + + pdc = PDC.return_value + pdc.get_latest_modules.return_value = [mod2_r1, mod3_r1] + + conf.git_base_url = "git://pkgs.fedoraproject.org" + utils.bump_distgit_repo.side_effect = [ + "fae7848fa47a854f25b782aa64441040a6d86544", + "43ec03000d249231bc7135b11b810afc96e90efb", + ] + + handler = MBSModuleStateChangeHandler() + handler.build_module = mock.Mock() + handler.build_module.side_effect = [123, 456] + + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + self.assertEqual(handler.build_module.call_args_list, + [mock.call('testmodule2', 'master', 'fae7848fa47a854f25b782aa64441040a6d86544'), + mock.call('testmodule3', 'master', '43ec03000d249231bc7135b11b810afc96e90efb')]) + + event_list = models.Event.query.all() + self.assertEqual(len(event_list), 1) + self.assertEqual(event_list[0].message_id, event.msg_id) + builds = models.ArtifactBuild.query.all() + self.assertEqual(len(builds), 2) + self.assertEqual(builds[0].name, mod2_r1['variant_name']) + self.assertEqual(builds[0].type, models.ARTIFACT_TYPES['module']) + self.assertEqual(builds[0].build_id, 123) + self.assertEqual(builds[1].name, mod3_r1['variant_name']) + self.assertEqual(builds[1].build_id, 456) + self.assertEqual(builds[1].type, models.ARTIFACT_TYPES['module']) + + @mock.patch('freshmaker.handlers.mbs.module_state_change.PDC') + @mock.patch('freshmaker.handlers.mbs.module_state_change.utils') + @mock.patch('freshmaker.handlers.conf') + def test_module_is_not_allowed_in_whitelist(self, conf, utils, PDC): + conf.handler_build_whitelist = { + "MBSModuleStateChangeHandler": { + "MBSModuleStateChangeEvent": { + "module": [ + { + 'name': 'base.*', + }, + ], + }, + }, + } + conf.handler_build_blacklist = {} + + msg = helpers.ModuleStateChangeMessage('testmodule', 'master', state='ready').produce() + event = self.get_event_from_msg(msg) + + mod2_info = helpers.PDCModuleInfo('testmodule2', 'master', '20170412010101') + mod2_info.add_build_dep('testmodule', 'master') + mod2 = mod2_info.produce() + + pdc = PDC.return_value + pdc.get_latest_modules.return_value = [mod2] + + handler = MBSModuleStateChangeHandler() + handler.build_module = mock.Mock() + handler.record_build = mock.Mock() + + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + handler.build_module.assert_not_called() + + @mock.patch('freshmaker.handlers.mbs.module_state_change.PDC') + @mock.patch('freshmaker.handlers.mbs.module_state_change.utils') + @mock.patch('freshmaker.handlers.conf') + def test_module_is_not_allowed_in_blacklist(self, conf, utils, PDC): + conf.handler_build_whitelist = {} + conf.handler_build_blacklist = { + "MBSModuleStateChangeHandler": { + "MBSModuleStateChangeEvent": { + "module": [ + { + 'name': 'test.*', + }, + ], + }, + }, + } + msg = helpers.ModuleStateChangeMessage('testmodule', 'master', state='ready').produce() + event = self.get_event_from_msg(msg) + + mod2_info = helpers.PDCModuleInfo('testmodule2', 'master', '20170412010101') + mod2_info.add_build_dep('testmodule', 'master') + mod2 = mod2_info.produce() + + pdc = PDC.return_value + pdc.get_latest_modules.return_value = [mod2] + + handler = MBSModuleStateChangeHandler() + handler.build_module = mock.Mock() + handler.record_build = mock.Mock() + + self.assertTrue(handler.can_handle(event)) + handler.handle(event) + + handler.build_module.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_models.py b/tests/test_models.py index 3d89fd9..e917ef7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -46,17 +46,17 @@ class TestModels(unittest.TestCase): db.session.expire_all() e = db.session.query(Event).filter(event.id == 1).one() - self.assertEquals(e.message_id, "test_msg_id") - self.assertEquals(len(e.builds), 2) - - self.assertEquals(e.builds[0].name, "ed") - self.assertEquals(e.builds[0].type, 2) - self.assertEquals(e.builds[0].state, 0) - self.assertEquals(e.builds[0].build_id, 1234) - self.assertEquals(e.builds[0].dep_of, None) - - self.assertEquals(e.builds[1].name, "mksh") - self.assertEquals(e.builds[1].type, 2) - self.assertEquals(e.builds[1].state, 0) - self.assertEquals(e.builds[1].build_id, 1235) - self.assertEquals(e.builds[1].dep_of.name, "ed") + self.assertEqual(e.message_id, "test_msg_id") + self.assertEqual(len(e.builds), 2) + + self.assertEqual(e.builds[0].name, "ed") + self.assertEqual(e.builds[0].type, 2) + self.assertEqual(e.builds[0].state, 0) + self.assertEqual(e.builds[0].build_id, 1234) + self.assertEqual(e.builds[0].dep_of, None) + + self.assertEqual(e.builds[1].name, "mksh") + self.assertEqual(e.builds[1].type, 2) + self.assertEqual(e.builds[1].state, 0) + self.assertEqual(e.builds[1].build_id, 1235) + self.assertEqual(e.builds[1].dep_of.name, "ed") From f343de129191bbcd3cffae90e73d3fe0861ffeb9 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: May 31 2017 07:56:01 +0000 Subject: [PATCH 2/2] Add load_class to replace fedmsg.utils.load_class handlers specified in config with format like: 'freshmaker.handlers.bodhi:BodhiUpdateCompleteStableHandler' doesn't work with fedmsg.utils.load_class under python3, add our own load_class to workaround this issue. --- diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 1e71927..acb611a 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -23,17 +23,31 @@ import abc import re -import fedmsg.utils +import sys from freshmaker import conf, log, db, models from freshmaker.mbs import MBS from freshmaker.kojiservice import koji_service +def load_class(location): + """ Take a string of the form 'fedmsg.consumers.ircbot:IRCBotConsumer' + and return the IRCBotConsumer class. + """ + mod_name, cls_name = location.strip().split(':') + + __import__(mod_name) + + try: + return getattr(sys.modules[mod_name], cls_name) + except AttributeError: + raise ImportError("%r not found in %r" % (cls_name, mod_name)) + + def load_handlers(): """ Import and instantiate all handlers listed in the given config. """ for import_path in conf.handlers: - cls = fedmsg.utils.load_class(import_path) + cls = load_class(import_path) handler = cls() yield handler