From ace63cf2f6c9c640115012cd9ef5ae742818056c Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Aug 01 2019 13:33:48 +0000 Subject: Remove unused and unmaintained parsers and handlers. We originally thought Freshmaker will be used to handle modules and container in the Fedora, but we have never reached that goal. All this code has been written 3 years ago and nobody touched it since that. It also most likely does not work at all, because modularity changed dramatically since that. I'm therefore removing this old code: - MBS and PDC classes. - All the modules related handlers and parser. - All the Git related handlers and parsers. - The pdc_* and mbs_* configuration fields. --- diff --git a/conf/config.py b/conf/config.py index 39ca037..991c518 100644 --- a/conf/config.py +++ b/conf/config.py @@ -31,9 +31,6 @@ class BaseConfiguration(object): NET_RETRY_INTERVAL = 30 SYSTEM = 'koji' - PDC_URL = 'http://modularity.fedorainfracloud.org:8080/rest_api/v1' - PDC_INSECURE = True - PDC_DEVELOP = True # Available backends are: console, file, journal. LOG_BACKEND = 'journal' @@ -49,18 +46,13 @@ class BaseConfiguration(object): # Parsers defined for parse specific messages PARSERS = [ 'freshmaker.parsers.internal:FreshmakerManualRebuildParser', - 'freshmaker.parsers.bodhi:BodhiUpdateCompleteStableParser', - 'freshmaker.parsers.git:GitReceiveParser', - 'freshmaker.parsers.mbs:MBSModuleStateChangeParser', + 'freshmaker.parsers.odcs:ComposeStateChangeParser', ] # List of enabled composing handlers. HANDLERS = [ - "freshmaker.handlers.koji:RebuildImagesOnRPMBodhiUpdate", - "freshmaker.handlers.koji:RebuildImagesOnGitDockerfileChange", - "freshmaker.handlers.mbs:RebuildModulesOnGitMMDChange", - "freshmaker.handlers.mbs:RebuildModulesOnGitRPMSpecChange", - "freshmaker.handlers.internal:UpdateDBOnModuleBuild", + 'freshmaker.handlers.internal:UpdateDBOnODCSComposeFail', + 'freshmaker.handlers.koji:RebuildImagesOnODCSComposeDone', ] # Base URL of git repository with source artifacts. @@ -72,15 +64,6 @@ class BaseConfiguration(object): # GIT user for cloning and pushing repo GIT_USER = "" - # Base URL of Module Build Service. - MBS_BASE_URL = "https://mbs.fedoraproject.org" - - # Authorization token to use when communicating with MBS. - MBS_AUTH_TOKEN = "" - - # PDC API URL - PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' - # Read Koji configuration from profile instead of reading them from # configuration file directly. For staging Koji, it is stg. KOJI_PROFILE = 'koji' @@ -286,12 +269,10 @@ class TestConfiguration(BaseConfiguration): MESSAGING = 'in_memory' MESSAGING_SENDER = 'in_memory' - PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' # Global network-related values, in seconds NET_TIMEOUT = 3 NET_RETRY_INTERVAL = 1 - MBS_AUTH_TOKEN = "testingtoken" KOJI_CONTAINER_SCRATCH_BUILD = True diff --git a/freshmaker/config.py b/freshmaker/config.py index 9bc4de1..8183a5c 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -121,18 +121,6 @@ class Config(object): 'type': bool, 'default': False, 'desc': 'Debug mode'}, - 'pdc_url': { - 'type': str, - 'default': '', - 'desc': 'PDC URL.'}, - 'pdc_insecure': { - 'type': bool, - 'default': False, - 'desc': 'Allow insecure connection to PDC.'}, - 'pdc_develop': { - 'type': bool, - 'default': False, - 'desc': 'PDC Development mode, basically noauth.'}, 'log_backend': { 'type': str, 'default': None, @@ -171,7 +159,7 @@ class Config(object): 'desc': 'Parsers defined for parse specific messages.'}, 'handlers': { 'type': list, - 'default': ["freshmaker.handlers.mbs:MBSModuleStateChangeHandler"], + 'default': [], 'desc': 'List of enabled handlers.'}, 'polling_interval': { 'type': int, @@ -193,14 +181,6 @@ class Config(object): 'type': str, 'default': 'Freshmaker ', 'desc': 'Author for git commit.'}, - 'mbs_base_url': { - 'type': str, - 'default': "https://mbs.fedoraproject.org", - 'desc': 'MBS Base URL'}, - 'mbs_auth_token': { - 'type': str, - 'default': '', - 'desc': "OpenIDC token to use when communicating with MBS."}, 'koji_profile': { 'type': str, 'default': 'koji', diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 79e1af8..dae76c1 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -30,7 +30,6 @@ from functools import wraps from freshmaker import conf, log, db, models from freshmaker.kojiservice import koji_service, parse_NVR -from freshmaker.mbs import MBS from freshmaker.models import ArtifactBuildState from freshmaker.types import EventState from freshmaker.models import ArtifactBuild, Event @@ -266,17 +265,6 @@ class BaseHandler(object): """ raise NotImplementedError() - def build_module(self, name, branch, rev): - """ - Build a module in MBS. - - :param name: module name. - :param branch: module branch. - :param rev: git revision. - """ - mbs = MBS(conf) - return mbs.build_module(name, branch, rev) - def record_build(self, event, name, artifact_type, build_id=None, dep_on=None, state=None, original_nvr=None, rebuilt_nvr=None, diff --git a/freshmaker/handlers/internal/__init__.py b/freshmaker/handlers/internal/__init__.py index c20cac8..f701278 100644 --- a/freshmaker/handlers/internal/__init__.py +++ b/freshmaker/handlers/internal/__init__.py @@ -20,7 +20,6 @@ # SOFTWARE. from .update_db_on_advisory_change import UpdateDBOnAdvisoryChange # noqa -from .update_db_on_module_build import UpdateDBOnModuleBuild # noqa from .generate_advisory_signed_event_on_rpm_sign import GenerateAdvisorySignedEventOnRPMSign # noqa from .update_db_on_odcs_compose_fail import UpdateDBOnODCSComposeFail # noqa from .cancel_event_on_freshmaker_manage_request import CancelEventOnFreshmakerManageRequest # noqa diff --git a/freshmaker/handlers/internal/update_db_on_module_build.py b/freshmaker/handlers/internal/update_db_on_module_build.py deleted file mode 100644 index aa0589d..0000000 --- a/freshmaker/handlers/internal/update_db_on_module_build.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - -from freshmaker import log, conf, utils, db, models -from freshmaker.types import ArtifactType, ArtifactBuildState -from freshmaker.mbs import MBS -from freshmaker.pdc import PDC -from freshmaker.handlers import BaseHandler, fail_event_on_handler_exception -from freshmaker.events import MBSModuleStateChangeEvent - - -class UpdateDBOnModuleBuild(BaseHandler): - name = "UpdateDBOnModuleBuild" - order = 0 - - def can_handle(self, event): - if isinstance(event, MBSModuleStateChangeEvent): - return True - - return False - - @fail_event_on_handler_exception - def handle(self, event): - """ - Update build state in db when module state changed in MBS and the - build is submitted by Freshmaker (can find that build in db). If - build state is 'ready', query PDC to get all modules that depends - on this module, rebuild all these depending modules. - """ - module_name = event.module - module_stream = event.stream - build_id = event.build_id - build_state = event.build_state - - module_build = None - # update build state if the build is submitted by Freshmaker - builds = db.session.query(models.ArtifactBuild).filter_by(build_id=build_id, - type=ArtifactType.MODULE.value).all() - if len(builds) > 1: - raise RuntimeError("Found duplicate module build '%s' in db" % build_id) - if len(builds) == 1: - # we can find this build in DB - module_build = builds.pop() - self.set_context(module_build) - if build_state in [MBS.BUILD_STATES['ready'], MBS.BUILD_STATES['failed']]: - log.info("Module build '%s' state changed in MBS, updating it in db.", build_id) - if build_state == MBS.BUILD_STATES['ready']: - module_build.state = ArtifactBuildState.DONE.value - if build_state == MBS.BUILD_STATES['failed']: - module_build.state = ArtifactBuildState.FAILED.value - db.session.commit() - - # Rebuild depending modules when state of MBSModuleStateChangeEvent is 'ready' - if build_state == MBS.BUILD_STATES['ready']: - log.info("Triggering rebuild of modules depending on %s:%s " - "in MBS", module_name, module_stream) - - if module_build: - # we have this build recorded in DB, check to prevent - # cyclic build loop - root_dep = module_build.get_root_dep_on() - if root_dep and root_dep.name == module_name: - log.info("Skipping the rebuild triggered by %s:%s as it will" - "result in cyclic build loop.", module_name, module_stream) - return [] - - pdc = PDC(conf) - modules = pdc.get_latest_modules(build_dep_name=module_name, - build_dep_stream=module_stream, - active='true') - - for mod in modules: - name = mod['name'] - version = mod['stream'] - if not self.allow_build(ArtifactType.MODULE, name=name, branch=version): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist", - name, version) - continue - # bump module repo first - commit_msg = "Bump to rebuild because of %s update" % module_name - rev = utils.bump_distgit_repo('modules', name, branch=version, commit_msg=commit_msg, logger=log) - new_build_id = self.build_module(name, version, rev) - if new_build_id is not None: - self.record_build(event, name, ArtifactType.MODULE, new_build_id, dep_on=module_build) - - return [] diff --git a/freshmaker/handlers/koji/__init__.py b/freshmaker/handlers/koji/__init__.py index 14a69fa..7f28a9b 100644 --- a/freshmaker/handlers/koji/__init__.py +++ b/freshmaker/handlers/koji/__init__.py @@ -19,8 +19,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from .rebuild_images_on_git_dockerfile_change import RebuildImagesOnGitDockerfileChange # noqa from .rebuild_images_on_odcs_compose_done import RebuildImagesOnODCSComposeDone # noqa from .rebuild_images_on_parent_image_build import RebuildImagesOnParentImageBuild # noqa from .rebuild_images_on_rpm_advisory_change import RebuildImagesOnRPMAdvisoryChange # noqa -from .rebuild_images_on_rpm_bodhi_update import RebuildImagesOnRPMBodhiUpdate # noqa diff --git a/freshmaker/handlers/koji/rebuild_images_on_git_dockerfile_change.py b/freshmaker/handlers/koji/rebuild_images_on_git_dockerfile_change.py deleted file mode 100644 index 16832fc..0000000 --- a/freshmaker/handlers/koji/rebuild_images_on_git_dockerfile_change.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -import koji - -from freshmaker import log, conf -from freshmaker.types import ArtifactType -from freshmaker.handlers import ContainerBuildHandler -from freshmaker.events import GitDockerfileChangeEvent - - -class RebuildImagesOnGitDockerfileChange(ContainerBuildHandler): - name = 'RebuildImagesOnGitDockerfileChange' - - def can_handle(self, event): - return isinstance(event, GitDockerfileChangeEvent) - - def handle(self, event): - """Rebuild docker image""" - log.info('Start to rebuild docker image %s.', event.container) - - if not self.allow_build(ArtifactType.IMAGE, name=event.container, branch=event.branch): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist", - event.container, event.branch) - return [] - - try: - name = event.container - branch = event.branch - rev = event.rev - scm_url = "{}/{}/{}.git?#{}".format( - conf.git_base_url, 'container', name, rev) - - build_target = '{}-container-candidate'.format( - 'rawhide' if branch == 'master' else branch) - - task_id = self.build_container(scm_url, branch, build_target) - if task_id is not None: - self.record_build(event, event.container, ArtifactType.IMAGE, task_id) - - except koji.krbV.Krb5Error as e: - log.exception('Failed to login Koji via Kerberos using GSSAPI. %s', e.args[1]) - except Exception: - log.exception('Could not create task to build docker image %s', event.container) - - return [] diff --git a/freshmaker/handlers/koji/rebuild_images_on_rpm_bodhi_update.py b/freshmaker/handlers/koji/rebuild_images_on_rpm_bodhi_update.py deleted file mode 100644 index 4fa0453..0000000 --- a/freshmaker/handlers/koji/rebuild_images_on_rpm_bodhi_update.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -from itertools import chain - -from freshmaker import conf -from freshmaker import log -from freshmaker import utils -from freshmaker.types import ArtifactType -from freshmaker.handlers import ContainerBuildHandler -from freshmaker.events import BodhiUpdateCompleteStableEvent -from freshmaker.pdc import PDC -from freshmaker.kojiservice import koji_service - - -class RebuildImagesOnRPMBodhiUpdate(ContainerBuildHandler): - """Rebuild docker images when RPMs are synced by Bodhi""" - name = 'RebuildImagesOnRPMBodhiUpdate' - - def can_handle(self, event): - return isinstance(event, BodhiUpdateCompleteStableEvent) - - def handle(self, event): - log.info('Rebuild docker images for event %s, msgid: %s', - BodhiUpdateCompleteStableEvent.__name__, event.msg_id) - - rpms = self.get_rpms_included_in_bodhi_update(event.builds) - containers = self.get_containers_including_rpms(rpms) - - log.info('Found docker images to rebuild: %s', containers) - - for container in containers: - if not self.allow_build(ArtifactType.IMAGE, name=container['name'], branch=container['branch']): - log.info("Skip rebuild of image %s:%s as it's not allowed by configured whitelist", - container['name'], container['branch']) - continue - try: - name = container['name'] - branch = container['branch'] - repo_url = '{}/{}/{}'.format(conf.git_base_url, 'container', name) - rev = utils.get_commit_hash(repo_url, branch=branch, logger=log) - - scm_url = "{}/{}/{}.git?#{}".format( - conf.git_base_url, 'container', name, rev) - - build_target = '{}-container-candidate'.format( - 'rawhide' if branch == 'master' else branch) - - task_id = self.build_container(scm_url, branch, build_target) - if task_id is not None: - self.record_build(event, container['name'], ArtifactType.IMAGE, task_id) - except Exception: - log.exception('Error when rebuild %s', container) - - return [] - - def get_rpms_included_in_bodhi_update(self, builds): - build_nvrs = (build['nvr'] for build in builds) - with koji_service(profile=conf.koji_profile, logger=log, - dry_run=conf.dry_run) as service: - return chain(*[service.get_build_rpms(nvr) for nvr in build_nvrs]) - - def get_containers_including_rpms(self, rpms): - containers = {} - pdc = PDC(conf) - for rpm in rpms: - found = pdc.find_containers_by_rpm_name(rpm['name']) - for container in found: - id = container['id'] - if id not in containers: - container_detail = pdc.get_release_component_by_id(id) - container['branch'] = container_detail['dist_git_branch'] - containers[id] = container - - return containers.values() diff --git a/freshmaker/handlers/mbs/__init__.py b/freshmaker/handlers/mbs/__init__.py deleted file mode 100644 index 4997bfe..0000000 --- a/freshmaker/handlers/mbs/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from .rebuild_modules_on_git_mmd_change import RebuildModulesOnGitMMDChange # noqa -from .rebuild_modules_on_git_rpm_spec_change import RebuildModulesOnGitRPMSpecChange # noqa diff --git a/freshmaker/handlers/mbs/rebuild_modules_on_git_mmd_change.py b/freshmaker/handlers/mbs/rebuild_modules_on_git_mmd_change.py deleted file mode 100644 index 4f62b90..0000000 --- a/freshmaker/handlers/mbs/rebuild_modules_on_git_mmd_change.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - - -from freshmaker import log -from freshmaker.types import ArtifactType -from freshmaker.handlers import BaseHandler -from freshmaker.events import GitModuleMetadataChangeEvent - - -class RebuildModulesOnGitMMDChange(BaseHandler): - name = "RebuildModulesOnGitMMDChange" - - def can_handle(self, event): - if isinstance(event, GitModuleMetadataChangeEvent): - return True - - return False - - def handle(self, event): - log.info("Triggering rebuild of module %s:%s, metadata updated (%s).", - event.module, event.branch, event.rev) - if not self.allow_build(ArtifactType.MODULE, name=event.module, branch=event.branch): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist", - event.module, event.branch) - return [] - - build_id = self.build_module(event.module, event.branch, event.rev) - if build_id is not None: - self.record_build(event, event.module, ArtifactType.MODULE, build_id) - - return [] diff --git a/freshmaker/handlers/mbs/rebuild_modules_on_git_rpm_spec_change.py b/freshmaker/handlers/mbs/rebuild_modules_on_git_rpm_spec_change.py deleted file mode 100644 index fe4a0cb..0000000 --- a/freshmaker/handlers/mbs/rebuild_modules_on_git_rpm_spec_change.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from freshmaker import log, conf, utils -from freshmaker.types import ArtifactType -from freshmaker.pdc import PDC -from freshmaker.handlers import BaseHandler -from freshmaker.events import GitRPMSpecChangeEvent - - -class RebuildModulesOnGitRPMSpecChange(BaseHandler): - name = "RebuildModulesOnGitRPMSpecChange" - - def can_handle(self, event): - if isinstance(event, GitRPMSpecChangeEvent): - return True - - return False - - def handle(self, event): - """ - Rebuild module when spec file of rpm in module is updated. - """ - rpm = event.rpm - branch = event.branch - rev = event.rev - - log.info("Triggering rebuild of modules on event of rpm (%s:%s) spec updated, rev: %s.", - rpm, branch, rev) - - pdc = PDC(conf) - modules = pdc.get_latest_modules(component_name=rpm, - component_branch=branch, - active='true') - - for module in modules: - name = module['name'] - version = module['stream'] - if not self.allow_build(ArtifactType.MODULE, name=name, branch=version): - log.info("Skip rebuild of %s:%s as it's not allowed by configured whitelist", - name, version) - continue - log.info("Going to rebuild module '%s:%s'.", name, version) - commit_msg = "Bump to rebuild because of %s rpm spec update (%s)." % (rpm, rev) - rev = utils.bump_distgit_repo('modules', name, branch=version, commit_msg=commit_msg, logger=log) - build_id = self.build_module(name, version, rev) - if build_id is not None: - self.record_build(event, name, ArtifactType.MODULE, build_id) - - return [] diff --git a/freshmaker/mbs.py b/freshmaker/mbs.py deleted file mode 100644 index ccb74ee..0000000 --- a/freshmaker/mbs.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - -import requests - -from freshmaker import log - - -class MBS(object): - BUILD_STATES = { - "init": 0, - "wait": 1, - "build": 2, - "done": 3, - "failed": 4, - "ready": 5, - } - - def __init__(self, config): - """ - :param config: config which has mbs_base_url, mbs_auth_token and git_base_url. - can just be an instance of freshmaker.config.Config - """ - self.base_url = config.mbs_base_url - self.auth_token = config.mbs_auth_token - self.git_base_url = config.git_base_url - - def build_module(self, name, branch, rev): - """ - Build module defined by name, branch and rev in MBS. - - :param name: module name. - :param branch: module branch. - :param rev: git revision. - :return: Build id or None in case of error. - """ - scm_url = self.git_base_url + '/modules/%s.git?#%s' % (name, rev) - - headers = {} - headers["Authorization"] = "Bearer %s" % self.auth_token - - body = {'scmurl': scm_url, 'branch': branch} - url = "%s/module-build-service/1/module-builds/" % self.base_url - - resp = requests.request("POST", url, headers=headers, json=body) - data = resp.json() - if data['status'] == 409: - # MBS HTTP 409 Conflict - The submitted module's NVR already exists. - # if module has been built already, we should just ignore it - log.info("Module (%s) is already exists in MBS", scm_url) - return None - - if 'id' in data: - log.info("Triggered build of %s, MBS build_id=%s", scm_url, data['id']) - return data['id'] - else: - log.error("Error when triggering build of %s: %s", scm_url, data) - - return None diff --git a/freshmaker/parsers/bodhi/__init__.py b/freshmaker/parsers/bodhi/__init__.py deleted file mode 100644 index 56f11f4..0000000 --- a/freshmaker/parsers/bodhi/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from .update_complete_stable import BodhiUpdateCompleteStableParser # noqa diff --git a/freshmaker/parsers/bodhi/update_complete_stable.py b/freshmaker/parsers/bodhi/update_complete_stable.py deleted file mode 100644 index b2ab173..0000000 --- a/freshmaker/parsers/bodhi/update_complete_stable.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -from freshmaker import log -from freshmaker.parsers import BaseParser -from freshmaker.events import BodhiUpdateCompleteStableEvent - - -class BodhiUpdateCompleteStableParser(BaseParser): - """Parse Bodhi message from topic bodhi.update.complete.stable""" - - name = 'BodhiUpdateCompleteStableParser' - topic_suffixes = ['bodhi.update.complete.stable'] - - def can_parse(self, topic, msg): - return any([topic.endswith(suffix) for suffix in self.topic_suffixes]) - - def parse(self, topic, msg): - msg_id = msg.get('msg_id') - msg_inner_msg = msg.get('msg') - - if not msg_inner_msg: - log.debug('Skipping message without any content with the topic "%s"', topic) - return None - - update = msg_inner_msg['update'] - return BodhiUpdateCompleteStableEvent(msg_id, - update_id=update['updateid'], - builds=update['builds'], - release=update['release']) diff --git a/freshmaker/parsers/git/__init__.py b/freshmaker/parsers/git/__init__.py deleted file mode 100644 index a57f1ad..0000000 --- a/freshmaker/parsers/git/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from .receive import GitReceiveParser # noqa diff --git a/freshmaker/parsers/git/receive.py b/freshmaker/parsers/git/receive.py deleted file mode 100644 index 0eb2951..0000000 --- a/freshmaker/parsers/git/receive.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - -from freshmaker import log -from freshmaker.parsers import BaseParser -from freshmaker.events import GitDockerfileChangeEvent -from freshmaker.events import GitModuleMetadataChangeEvent -from freshmaker.events import GitRPMSpecChangeEvent - - -class GitReceiveParser(BaseParser): - """ - Parser parsing message from dist-git. - """ - name = "GitReceiveParser" - topic_suffixes = ["git.receive"] - - def can_parse(self, topic, msg): - if not any([topic.endswith(s) for s in self.topic_suffixes]): - return False - return True - - def parse(self, topic, msg): - msg_id = msg.get('msg_id') - msg_inner_msg = msg.get('msg') - - # If there isn't a msg dict in msg then this message can be skipped - if not msg_inner_msg: - log.debug(('Skipping message without any content with the ' - 'topic "{0}"').format(topic)) - return None - - commit = msg_inner_msg.get('commit') - if not commit: - log.debug(('Skipping message without commit with the ' - 'topic "{0}"').format(topic)) - return None - - namespace = commit.get("namespace") - repo = commit.get("repo") - branch = commit.get("branch") - rev = commit.get("rev") - - log.debug(namespace) - - if namespace == "modules": - changed_files = commit.get('stats', {}).get('files', {}).keys() - metadata_changed = '%s.yaml' % repo in changed_files - if metadata_changed: - log.debug("Parsed GitModuleMetadataChangeEvent fedmsg, repo=%s, " - "branch=%s, rev=%s", repo, branch, rev) - return GitModuleMetadataChangeEvent(msg_id, repo, branch, rev) - - elif namespace == 'container': - changed_files = msg['msg']['commit']['stats']['files'] - if 'Dockerfile' in changed_files: - log.debug("Parsed GitDockerfileChangeEvent fedmsg, repo=%s, " - "branch=%s, rev=%s", repo, branch, rev) - return GitDockerfileChangeEvent(msg_id, repo, branch, rev) - - elif namespace == 'rpms': - changed_files = commit.get('stats', {}).get('files', {}).keys() - has_spec = any([i.endswith('.spec') for i in changed_files]) - if has_spec: - log.debug("Parsed GitRPMSpecChangeEvent fedmsg, repo=%s, " - "branch=%s, rev=%s", repo, branch, rev) - return GitRPMSpecChangeEvent(msg_id, repo, branch, rev) - - return None diff --git a/freshmaker/parsers/mbs/__init__.py b/freshmaker/parsers/mbs/__init__.py deleted file mode 100644 index 2a01c62..0000000 --- a/freshmaker/parsers/mbs/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from .module_state_change import MBSModuleStateChangeParser # noqa diff --git a/freshmaker/parsers/mbs/module_state_change.py b/freshmaker/parsers/mbs/module_state_change.py deleted file mode 100644 index a647084..0000000 --- a/freshmaker/parsers/mbs/module_state_change.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza - -from freshmaker import log -from freshmaker.parsers import BaseParser -from freshmaker.events import MBSModuleStateChangeEvent - - -class MBSModuleStateChangeParser(BaseParser): - """ - Parser parsing message from module-build-service, generating - ModuleBuilt event. - """ - name = "MBSModuleStateChangeParser" - topic_suffixes = ["mbs.module.state.change"] - - def can_parse(self, topic, msg): - log.debug(topic) - if not any([topic.endswith(s) for s in self.topic_suffixes]): - return False - return True - - def parse(self, topic, msg): - msg_id = msg.get('msg_id') - msg_inner_msg = msg.get('msg') - - # If there isn't a msg dict in msg then this message can be skipped - if not msg_inner_msg: - log.debug(('Skipping message without any content with the ' - 'topic "{0}"').format(topic)) - return None - - return MBSModuleStateChangeEvent(msg_id, - msg_inner_msg.get('name'), - msg_inner_msg.get('stream'), - msg_inner_msg.get('id'), - msg_inner_msg.get('state')) diff --git a/freshmaker/pdc.py b/freshmaker/pdc.py deleted file mode 100644 index 7fff388..0000000 --- a/freshmaker/pdc.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - -import inspect -import requests -import concurrent.futures -from itertools import groupby -from pdc_client import PDCClient -import freshmaker.utils - - -class PDC(object): - def __init__(self, config): - # pdc_url, pdc_develop and pdc_insecure should be avaiable in config - self.config = config - self.session = self.get_client_session() - - def get_client_session(self): - """ - Return pdc_client.PDCClient instance - """ - if 'ssl_verify' in inspect.getargspec(PDCClient.__init__).args: - # New API - return PDCClient( - server=self.config.pdc_url, - develop=self.config.pdc_develop, - ssl_verify=not self.config.pdc_insecure, - ) - else: - # Old API - return PDCClient( - server=self.config.pdc_url, - develop=self.config.pdc_develop, - insecure=self.config.pdc_insecure, - ) - - def is_latest_module(self, module): - """Check if given module is the latest one in the name:stream in PDC""" - data = self.get_modules( - name=module['name'], - stream=module['stream'], - fields='version', - ordering='-version', - page_size=1) - return data['results'][0]['version'] == module['version'] - - def get_latest_modules(self, **criteria): - criteria.update({ - 'fields': ['uid', 'name', 'stream', 'version'], - 'ordering': 'name,stream,-version', - }) - modules = self.get_modules(**criteria) - - def _return_module_if_latest(module): - return module if self.is_latest_module(module) else None - - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - futures = [ - executor.submit(_return_module_if_latest, - list(stream_modules)[0]) - for name_stream, stream_modules in groupby( - modules, key=lambda m: '%(name)s:%(stream)s' % m) - ] - concurrent.futures.wait(futures) - - return [m for m in (f.result() for f in futures) if m] - - @freshmaker.utils.retry(wait_on=(requests.Timeout, requests.ConnectionError), logger=freshmaker.log) - def get_modules(self, **kwargs): - """ - Query PDC with specified query parameters and return a list of modules. - - :param kwargs: query parameters in keyword arguments - :return: a list of modules - """ - page_size = kwargs.pop('page_size', -1) - modules = self.session['modules'](page_size=page_size, **kwargs) - return modules - - @freshmaker.utils.retry(wait_on=(requests.Timeout, requests.ConnectionError), logger=freshmaker.log) - def find_containers_by_rpm_name(self, rpm_name): - rels = self.session['release-component-relationships'](type='ContainerIncludesRPM', - to_component_name=rpm_name) - return [rel['from_component'] for rel in rels['results']] - - @freshmaker.utils.retry(wait_on=(requests.Timeout, requests.ConnectionError), logger=freshmaker.log) - def get_release_component_by_id(self, id): - return self.session['release-components/{}/'.format(id)]() diff --git a/tests/handlers/internal/test_update_db_on_module_build.py b/tests/handlers/internal/test_update_db_on_module_build.py deleted file mode 100644 index 64630b3..0000000 --- a/tests/handlers/internal/test_update_db_on_module_build.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import sys -import unittest -import mock - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa -from tests import helpers - -import freshmaker - -from freshmaker import events, db, models -from freshmaker.types import ArtifactType -from freshmaker.handlers.internal import UpdateDBOnModuleBuild -from freshmaker.parsers.mbs import MBSModuleStateChangeParser -from freshmaker.config import any_ - - -class UpdateDBOnModuleBuildTest(helpers.ModelsTestCase): - def setUp(self): - super(UpdateDBOnModuleBuildTest, self).setUp() - events.BaseEvent.register_parser(MBSModuleStateChangeParser) - - def test_can_handle_module_state_change_event(self): - """ - Tests MBS handler can handle module built message - """ - for state in ['init', 'wait', 'build', 'done', 'failed', 'ready']: - msg = helpers.ModuleStateChangeMessage('testmodule', 'master', state=state).produce() - event = self.get_event_from_msg(msg) - - handler = UpdateDBOnModuleBuild() - self.assertTrue(handler.can_handle(event)) - - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.PDC') - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.utils') - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.conf') - @mock.patch.object(freshmaker.conf, 'handler_build_whitelist', new={ - 'UpdateDBOnModuleBuild': { - 'module': any_({'name': r'testmodule\d*'}, {'branch': 'master'}), - } - }) - def test_can_rebuild_depending_modules(self, conf, utils, PDC): - """ - Tests handler can rebuild all modules which depend on the module - in module state change event. - """ - msg = helpers.ModuleStateChangeMessage('testmodule', 'master', state='ready').produce() - event = self.get_event_from_msg(msg) - - mod2_r1_info = helpers.PDCModuleInfo('testmodule2', 'master', '20170412010101') - mod2_r1_info.add_build_dep('testmodule', 'master') - mod2_r1 = mod2_r1_info.produce() - - mod3_r1_info = helpers.PDCModuleInfo('testmodule3', 'master', '20170412010201') - mod3_r1_info.add_build_dep('testmodule', 'master') - mod3_r1 = mod3_r1_info.produce() - - pdc = PDC.return_value - pdc.get_latest_modules.return_value = [mod2_r1, mod3_r1] - - conf.git_base_url = "git://pkgs.fedoraproject.org" - utils.bump_distgit_repo.side_effect = [ - "fae7848fa47a854f25b782aa64441040a6d86544", - "43ec03000d249231bc7135b11b810afc96e90efb", - ] - - handler = UpdateDBOnModuleBuild() - handler.build_module = mock.Mock() - handler.build_module.side_effect = [123, 456] - - self.assertTrue(handler.can_handle(event)) - handler.handle(event) - - self.assertEqual(handler.build_module.call_args_list, - [mock.call('testmodule2', 'master', 'fae7848fa47a854f25b782aa64441040a6d86544'), - mock.call('testmodule3', 'master', '43ec03000d249231bc7135b11b810afc96e90efb')]) - - event_list = models.Event.query.all() - self.assertEqual(len(event_list), 1) - self.assertEqual(event_list[0].message_id, event.msg_id) - builds = models.ArtifactBuild.query.all() - self.assertEqual(len(builds), 2) - self.assertEqual(builds[0].name, mod2_r1['name']) - self.assertEqual(builds[0].type, ArtifactType.MODULE.value) - self.assertEqual(builds[0].build_id, 123) - self.assertEqual(builds[1].name, mod3_r1['name']) - self.assertEqual(builds[1].build_id, 456) - self.assertEqual(builds[1].type, ArtifactType.MODULE.value) - - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.PDC') - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.utils') - @mock.patch('freshmaker.handlers.conf') - def test_module_is_not_allowed_in_whitelist(self, conf, utils, PDC): - conf.handler_build_whitelist = { - "UpdateDBOnModuleBuild": { - "module": any_( - { - 'name': 'base.*', - }, - ), - }, - } - - msg = helpers.ModuleStateChangeMessage('testmodule', 'master', state='ready').produce() - event = self.get_event_from_msg(msg) - - mod2_info = helpers.PDCModuleInfo('testmodule2', 'master', '20170412010101') - mod2_info.add_build_dep('testmodule', 'master') - mod2 = mod2_info.produce() - - pdc = PDC.return_value - pdc.get_latest_modules.return_value = [mod2] - - handler = UpdateDBOnModuleBuild() - handler.build_module = mock.Mock() - handler.record_build = mock.Mock() - - self.assertTrue(handler.can_handle(event)) - handler.handle(event) - - handler.build_module.assert_not_called() - - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.PDC') - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.utils') - @mock.patch('freshmaker.handlers.internal.update_db_on_module_build.log') - @mock.patch.object(freshmaker.conf, 'handler_build_whitelist', new={ - 'UpdateDBOnModuleBuild': { - 'module': any_({'name': r'module\d+'}, {'branch': 'master'}) - } - }) - def test_handler_not_fall_into_cyclic_rebuild_loop(self, log, utils, PDC): - """ - Tests handler will not fall into cyclic rebuild loop when there is - build dep loop of modules. - """ - # in this case, we have: - # 1. module2 depends on module1 - # 2. module3 depends on module2 - # 3. module1 depends on module3 - # - # when we receives a modult built event of module1, the expect result is: - # 1. module2 get rebuild because module1 is built - # 2. module3 get rebuild because module2 is built - # 3. module1 get rebuild because module3 is built - # 4. stop here - - utils.bump_distgit_repo.return_value = 'abcd' - - mod1_info = helpers.PDCModuleInfo('module1', 'master', '20170412010101') - mod1_info.add_build_dep('module3', 'master') - mod1 = mod1_info.produce() - - mod2_info = helpers.PDCModuleInfo('module2', 'master', '20170412010102') - mod2_info.add_build_dep('module1', 'master') - mod2 = mod2_info.produce() - - mod3_info = helpers.PDCModuleInfo('module3', 'master', '20170412010103') - mod3_info.add_build_dep('module2', 'master') - mod3 = mod3_info.produce() - - pdc = PDC.return_value - handler = UpdateDBOnModuleBuild() - - # Assume we have build of module1 recorded in DB already, it doesn't has - # any dep_on as it was initial triggered by an event which is not - # associated with any build in our DB. - event = models.Event.create(db.session, "initial_msg_id", "test", events.TestingEvent) - models.ArtifactBuild.create(db.session, event, "module1", "module", '123') - db.session.commit() - - # we received module built event of module1 - msg = helpers.ModuleStateChangeMessage('module1', 'master', state='ready', build_id=123).produce() - event = self.get_event_from_msg(msg) - pdc.get_latest_modules.return_value = [mod2] - handler.build_module = mock.Mock() - handler.build_module.return_value = 124 - - # this will trigger module rebuild of module2 - handler.handle(event) - handler.build_module.assert_called_once_with('module2', 'master', 'abcd') - - # we received module built event of module2 - msg = helpers.ModuleStateChangeMessage('module2', 'master', state='ready', build_id=124).produce() - event = self.get_event_from_msg(msg) - pdc.get_latest_modules.return_value = [mod3] - handler.build_module = mock.Mock() - handler.build_module.return_value = 125 - - # this will trigger module rebuild of module3 - handler.handle(event) - handler.build_module.assert_called_once_with('module3', 'master', 'abcd') - - # we received module built event of module3 - msg = helpers.ModuleStateChangeMessage('module3', 'master', state='ready', build_id=125).produce() - event = self.get_event_from_msg(msg) - pdc.get_latest_modules.return_value = [mod1] - handler.build_module = mock.Mock() - handler.build_module.return_value = 126 - - # this will trigger module rebuild of module1 - handler.handle(event) - handler.build_module.assert_called_once_with('module1', 'master', 'abcd') - - # we received module built event of module1 - msg = helpers.ModuleStateChangeMessage('module1', 'master', state='ready', build_id=126).produce() - event = self.get_event_from_msg(msg) - pdc.get_latest_modules.return_value = [mod2] - handler.build_module = mock.Mock() - - # but this time we should not rebuild module2 - handler.handle(event) - handler.build_module.assert_not_called() - log.info.assert_has_calls([mock.call('Skipping the rebuild triggered by %s:%s as it willresult in cyclic build loop.', 'module1', u'master')]) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/handlers/koji/test_rebuild_images_on_git_dockerfile_change.py b/tests/handlers/koji/test_rebuild_images_on_git_dockerfile_change.py deleted file mode 100644 index ccb8a74..0000000 --- a/tests/handlers/koji/test_rebuild_images_on_git_dockerfile_change.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -from mock import patch -from mock import PropertyMock - -import freshmaker - -from freshmaker import models -from freshmaker.types import ArtifactType -from freshmaker.config import any_ -from tests import get_fedmsg, helpers - - -class BaseTestCase(helpers.ModelsTestCase): - - @patch("requests.request") - @patch('freshmaker.consumer.get_global_consumer') - def consume_fedmsg(self, msg, global_consumer, request): - consumer = self.create_consumer() - global_consumer.return_value = consumer - consumer.consume(msg) - - -class RebuildImagesOnGitDockerfileChangeTest(BaseTestCase): - - @patch('koji.read_config') - @patch('koji.ClientSession') - @patch("freshmaker.config.Config.krb_auth_principal", - new_callable=PropertyMock, return_value="user@example.com") - @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ - 'RebuildImagesOnGitDockerfileChange': { - 'image': any_({'name': 'testimage'}, {'branch': 'master'}) - } - }) - def test_rebuild_if_dockerfile_changed( - self, auth_principal, ClientSession, read_config): - read_config.return_value = { - 'server': 'https://localhost/kojihub', - 'krb_rdns': False, - 'weburl': 'https://localhost/koji', - } - - mock_session = ClientSession.return_value - mock_session.buildContainer.return_value = 123 - msg = get_fedmsg('git_receive_dockerfile_changed') - self.consume_fedmsg(msg) - - mock_session.krb_login.assert_called() - mock_session.buildContainer.assert_called_once_with( - 'git://pkgs.fedoraproject.org/container/testimage.git?#e1f39d43471fc37ec82616f76a119da4eddec787', - 'rawhide-container-candidate', - {'scratch': True, 'git_branch': 'master'}) - mock_session.logout.assert_called_once() - - events = models.Event.query.all() - self.assertEqual(len(events), 1) - self.assertEqual(events[0].message_id, msg['body']['msg_id']) - builds = models.ArtifactBuild.query.all() - self.assertEqual(len(builds), 1) - self.assertEqual(builds[0].name, 'testimage') - self.assertEqual(builds[0].type, ArtifactType.IMAGE.value) - self.assertEqual(builds[0].build_id, 123) - - @patch('freshmaker.handlers.koji.RebuildImagesOnGitDockerfileChange.build_container') - def test_not_rebuild_if_dockerfile_not_changed(self, build_container): - self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_not_changed')) - build_container.assert_not_called() - - @patch('koji.read_config') - @patch('koji.ClientSession') - @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ - 'RebuildImagesOnGitDockerfileChange': { - 'image': any_({'name': 'testimage'}, {'branch': 'master'}) - } - }) - def test_ensure_logout_in_whatever_case(self, ClientSession, read_config): - ClientSession.return_value.buildContainer.side_effect = RuntimeError - read_config.return_value = { - 'server': 'https://localhost/kojihub', - 'krb_rdns': False, - 'weburl': 'https://localhost/koji', - } - - self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) - - ClientSession.return_value.logout.assert_called_once() - - @patch('koji.read_config') - @patch('koji.ClientSession') - @patch("freshmaker.config.Config.krb_auth_principal", - new_callable=PropertyMock, return_value="user@example.com") - def test_ensure_do_nothing_if_fail_to_login_koji(self, auth_principal, ClientSession, read_config): - ClientSession.return_value.krb_login.side_effect = RuntimeError - read_config.return_value = { - 'server': 'https://localhost/kojihub', - 'krb_rdns': False, - 'weburl': 'https://localhost/koji', - } - - self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) - - ClientSession.return_value.buildContainer.assert_not_called() diff --git a/tests/handlers/koji/test_rebuild_images_on_rpm_bodhi_update.py b/tests/handlers/koji/test_rebuild_images_on_rpm_bodhi_update.py deleted file mode 100644 index 18c5313..0000000 --- a/tests/handlers/koji/test_rebuild_images_on_rpm_bodhi_update.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Chenxiong Qi - -import mock -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa -from tests import helpers -from tests import get_fedmsg - -import freshmaker - -from freshmaker import events, models -from freshmaker.types import ArtifactType -from freshmaker.handlers.koji import RebuildImagesOnRPMBodhiUpdate -from freshmaker.parsers.bodhi import BodhiUpdateCompleteStableParser - -mock_found_containers = [ - { - 'release': 'fedora-25-updates', - 'id': 5430, - 'name': 'testimage1', - 'branch': 'f25', - }, - { - 'release': 'fedora-25-updates', - 'id': 5431, - 'name': 'testimage2', - 'branch': 'f25', - }, -] - -mock_release_components = { - 5430: { - 'id': 5430, - 'release': { - 'active': True, - 'release_id': 'fedora-25-updates' - }, - 'bugzilla_component': None, - 'brew_package': None, - 'global_component': 'testimage1', - 'name': 'testimage1', - 'dist_git_branch': 'f25', - 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage1', - 'active': True, - 'type': 'container', - 'srpm': None, - }, - 5431: { - 'id': 5431, - 'release': { - 'active': True, - 'release_id': 'fedora-25-updates' - }, - 'bugzilla_component': None, - 'brew_package': None, - 'global_component': 'testimage2', - 'name': 'testimage2', - 'dist_git_branch': 'f25', - 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage2', - 'active': True, - 'type': 'container', - 'srpm': None, - } -} - - -def mock_get_release_component_by_id(id): - return mock_release_components[id] - - -class RebuildImagesOnRPMBodhiUpdateTest(helpers.ModelsTestCase): - def setUp(self): - super(RebuildImagesOnRPMBodhiUpdateTest, self).setUp() - events.BaseEvent.register_parser(BodhiUpdateCompleteStableParser) - - @mock.patch('freshmaker.handlers.koji.rebuild_images_on_rpm_bodhi_update.PDC') - @mock.patch('freshmaker.handlers.koji.rebuild_images_on_rpm_bodhi_update.utils') - @mock.patch('freshmaker.handlers.koji.rebuild_images_on_rpm_bodhi_update.conf') - @mock.patch.object(freshmaker.conf, 'handler_build_whitelist', new={ - 'RebuildImagesOnRPMBodhiUpdate': { - 'image': {'name': r'testimage\d', 'branch': 'f25'} - } - }) - def test_trigger_rebuild_container_when_receives_bodhi_rebuild_images_on_rpm_bodhi_update_message(self, conf, utils, PDC): - - conf.git_base_url = 'git://pkgs.fedoraproject.org' - - handler = RebuildImagesOnRPMBodhiUpdate() - handler.get_rpms_included_in_bodhi_update = mock.Mock() - - handler.get_containers_including_rpms = mock.Mock() - - containers = [ - { - 'release': 'fedora-25-updates', - 'id': 5430, - 'name': 'testimage1', - 'branch': 'f25', - }, - { - 'release': 'fedora-25-updates', - 'id': 5431, - 'name': 'testimage2', - 'branch': 'f25', - }, - ] - handler.get_containers_including_rpms.return_value = containers - - utils.get_commit_hash.side_effect = ['c123', 'c456'] - - handler.build_container = mock.Mock() - handler.build_container.side_effect = [123, 456] - - msg = get_fedmsg('bodhi_update_stable') - event = self.get_event_from_msg(msg) - self.assertTrue(handler.can_handle(event)) - handler.handle(event) - - self.assertEqual( - handler.build_container.call_args_list, - [mock.call( - 'git://pkgs.fedoraproject.org/container/testimage1.git?#c123', - 'f25', 'f25-container-candidate'), - mock.call( - 'git://pkgs.fedoraproject.org/container/testimage2.git?#c456', - 'f25', 'f25-container-candidate')]) - - events = models.Event.query.all() - self.assertEqual(len(events), 1) - self.assertEqual(events[0].message_id, msg['body']['msg_id']) - builds = models.ArtifactBuild.query.all() - self.assertEqual(len(builds), 2) - self.assertEqual(builds[0].name, 'testimage1') - self.assertEqual(builds[0].type, ArtifactType.IMAGE.value) - self.assertEqual(builds[0].build_id, 123) - self.assertEqual(builds[1].name, 'testimage2') - self.assertEqual(builds[1].type, ArtifactType.IMAGE.value) - self.assertEqual(builds[1].build_id, 456) - - @mock.patch('freshmaker.handlers.koji.rebuild_images_on_rpm_bodhi_update.PDC') - @mock.patch('freshmaker.handlers.koji.rebuild_images_on_rpm_bodhi_update.utils') - @mock.patch('freshmaker.handlers.koji.rebuild_images_on_rpm_bodhi_update.conf') - def test_get_containers_including_rpms(self, conf, utils, PDC): - expected_found_containers = [ - { - 'release': 'fedora-24-updates', - 'id': 5430, - 'name': 'testimage1', - 'branch': 'f25', - }, - { - 'release': 'fedora-24-updates', - 'id': 5431, - 'name': 'testimage2', - 'branch': 'f25', - }, - ] - pdc = PDC(conf) - pdc.find_containers_by_rpm_name.return_value = expected_found_containers - - handler = RebuildImagesOnRPMBodhiUpdate() - rpms = [ - {'id': 9515683, - 'name': 'community-mysql-devel', - 'nvr': 'community-mysql-devel-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18'}, - {'id': 9515682, - 'name': 'community-mysql-libs', - 'nvr': 'community-mysql-libs-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18'}, - {'id': 9515681, - 'name': 'community-mysql-server', - 'nvr': 'community-mysql-server-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18'}, - ] - - containers = handler.get_containers_including_rpms(rpms) - - self.assertEqual(3, pdc.find_containers_by_rpm_name.call_count) - found_containers = sorted(containers, key=lambda item: item['id']) - self.assertEqual(expected_found_containers, found_containers) - - @helpers.mock_koji - def test_get_rpms_included_in_bohdhi_update(self, mocked_koji): - mocked_koji.add_build("community-mysql-5.7.18-2.fc25") - mocked_koji.add_build_rpms( - "community-mysql-5.7.18-2.fc25", - ['community-mysql-devel-5.7.18-2.fc25', - 'community-mysql-libs-5.7.18-2.fc25', - 'community-mysql-server-5.7.18-2.fc25'], ["i686"]) - - mocked_koji.add_build("qt5-qtwebengine-5.8.0-11.fc25") - mocked_koji.add_build_rpms( - "qt5-qtwebengine-5.8.0-11.fc25", - ['qt5-qtwebengine-devel-5.8.0-11.fc25', - 'qt5-qtwebengine-examples-5.8.0-11.fc25'], ["i686"]) - - builds = [ - { - 'build_id': 884455, - 'name': 'qt5-qtwebengine', - 'nvr': 'qt5-qtwebengine-5.8.0-11.fc25', - 'release': '11.fc25', - 'version': '5.8.0', - }, - { - 'build_id': 881597, - 'name': 'community-mysql', - 'nvr': 'community-mysql-5.7.18-2.fc25', - 'release': '2.fc25', - 'version': '5.7.18', - } - ] - handler = RebuildImagesOnRPMBodhiUpdate() - rpms = list(handler.get_rpms_included_in_bodhi_update(builds)) - - self.assertEqual(5, len(rpms)) - - rpm = list(filter(lambda item: item['nvr'] == "community-mysql-server-5.7.18-2.fc25", rpms)) - self.assertEqual(1, len(rpm)) diff --git a/tests/handlers/mbs/__init__.py b/tests/handlers/mbs/__init__.py deleted file mode 100644 index 043e3c6..0000000 --- a/tests/handlers/mbs/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Jan Kaluza diff --git a/tests/handlers/mbs/test_rebuild_modules_on_git_mmd_change.py b/tests/handlers/mbs/test_rebuild_modules_on_git_mmd_change.py deleted file mode 100644 index 67111e8..0000000 --- a/tests/handlers/mbs/test_rebuild_modules_on_git_mmd_change.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import sys -import unittest -import mock - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa -from tests import helpers - -import freshmaker - -from freshmaker import events, models -from freshmaker.types import ArtifactType -from freshmaker.handlers.mbs import RebuildModulesOnGitMMDChange -from freshmaker.parsers.git import GitReceiveParser -from freshmaker.config import any_ - - -class RebuildModulesOnGitMMDChangeTest(helpers.ModelsTestCase): - def setUp(self): - super(RebuildModulesOnGitMMDChangeTest, self).setUp() - events.BaseEvent.register_parser(GitReceiveParser) - - def test_can_handle_module_metadata_change_event(self): - """ - Tests handler can handle module metadata change message - """ - m = helpers.DistGitMessage('modules', 'testmodule', 'master', '123') - m.add_changed_file('testmodule.yaml', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - handler = RebuildModulesOnGitMMDChange() - self.assertTrue(handler.can_handle(event)) - - @mock.patch.object(freshmaker.conf, 'handler_build_whitelist', new={ - 'RebuildModulesOnGitMMDChange': { - 'module': any_({'name': 'testmodule'}, {'branch': 'master'}) - } - }) - def test_can_rebuild_module_when_module_metadata_changed(self): - """ - Tests handler can rebuild module when module metadata is changed in dist-git - """ - m = helpers.DistGitMessage('modules', 'testmodule', 'master', '12345') - m.add_changed_file('testmodule.yaml', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - handler = RebuildModulesOnGitMMDChange() - handler.build_module = mock.Mock() - handler.build_module.return_value = 123 - - self.assertTrue(handler.can_handle(event)) - handler.handle(event) - - self.assertEqual(handler.build_module.call_args_list, - [mock.call('testmodule', 'master', '12345')]) - - event_list = models.Event.query.all() - self.assertEqual(len(event_list), 1) - self.assertEqual(event_list[0].message_id, event.msg_id) - builds = models.ArtifactBuild.query.all() - self.assertEqual(len(builds), 1) - self.assertEqual(builds[0].name, 'testmodule') - self.assertEqual(builds[0].type, ArtifactType.MODULE.value) - self.assertEqual(builds[0].build_id, 123) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/handlers/mbs/test_rebuild_modules_on_git_rpm_spec_change.py b/tests/handlers/mbs/test_rebuild_modules_on_git_rpm_spec_change.py deleted file mode 100644 index 9b6e7ea..0000000 --- a/tests/handlers/mbs/test_rebuild_modules_on_git_rpm_spec_change.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2017 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import sys -import unittest -import mock - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa -from tests import helpers - -import freshmaker - -from freshmaker import events, models -from freshmaker.types import ArtifactType -from freshmaker.handlers.mbs import RebuildModulesOnGitRPMSpecChange -from freshmaker.parsers.git import GitReceiveParser -from freshmaker.config import any_ - - -class RebuildModulesOnGitRPMSpecChangeTest(helpers.ModelsTestCase): - def setUp(self): - super(RebuildModulesOnGitRPMSpecChangeTest, self).setUp() - events.BaseEvent.register_parser(GitReceiveParser) - - def test_can_handle_dist_git_message_with_rpm_spec_changed(self): - """ - Tests handler can handle rpm spec change event - """ - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('bash.spec', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - handler = RebuildModulesOnGitRPMSpecChange() - self.assertTrue(handler.can_handle(event)) - - def test_can_not_handle_dist_git_message_without_rpm_spec_changed(self): - """ - Tests can not handle dist git message that spec file is not changed. - """ - - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('test.c', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - handler = RebuildModulesOnGitRPMSpecChange() - self.assertFalse(handler.can_handle(event)) - - @mock.patch('freshmaker.handlers.mbs.rebuild_modules_on_git_rpm_spec_change.PDC') - @mock.patch('freshmaker.handlers.mbs.rebuild_modules_on_git_rpm_spec_change.utils') - @mock.patch('freshmaker.handlers.mbs.rebuild_modules_on_git_rpm_spec_change.conf') - @mock.patch.object(freshmaker.conf, 'handler_build_whitelist', new={ - 'RebuildModulesOnGitRPMSpecChange': { - 'module': any_({'name': 'testmodule'}, {'branch': 'master'}) - } - }) - def test_can_rebuild_modules_has_rpm_included(self, conf, utils, PDC): - """ - Test handler can rebuild modules which include the rpm. - """ - conf.git_base_url = "git://pkgs.fedoraproject.org" - - m = helpers.DistGitMessage('rpms', 'bash', 'master', '123') - m.add_changed_file('bash.spec', 1, 1) - msg = m.produce() - - event = self.get_event_from_msg(msg) - - mod_info = helpers.PDCModuleInfo('testmodule', 'master', '20170412010101') - mod_info.add_rpm("bash-1.2.3-4.f26.rpm") - mod = mod_info.produce() - pdc = PDC.return_value - pdc.get_latest_modules.return_value = [mod] - - commitid = '9287eb8eb4c4c60f73b4a59f228a673846d940c6' - utils.bump_distgit_repo.return_value = commitid - - handler = RebuildModulesOnGitRPMSpecChange() - handler.build_module = mock.Mock() - handler.build_module.return_value = 123 - - self.assertTrue(handler.can_handle(event)) - handler.handle(event) - - handler.build_module.assert_called_with('testmodule', 'master', commitid) - - event_list = models.Event.query.all() - self.assertEqual(len(event_list), 1) - self.assertEqual(event_list[0].message_id, event.msg_id) - builds = models.ArtifactBuild.query.all() - self.assertEqual(len(builds), 1) - self.assertEqual(builds[0].name, 'testmodule') - self.assertEqual(builds[0].type, ArtifactType.MODULE.value) - self.assertEqual(builds[0].build_id, 123) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_consumer.py b/tests/test_consumer.py index 7087ebd..5628d1f 100644 --- a/tests/test_consumer.py +++ b/tests/test_consumer.py @@ -33,15 +33,16 @@ from tests import helpers class ConsumerBaseTest(helpers.ModelsTestCase): - def _module_state_change_msg(self, state=None): + def _compose_state_change_msg(self, state=None): msg = {'body': { "msg_id": "2017-7afcb214-cf82-4130-92d2-22f45cf59cf7", - "topic": "org.fedoraproject.prod.mbs.module.state.change", + "topic": "org.fedoraproject.prod.odcs.state.change", "signature": "qRZ6oXBpKD/q8BTjBNa4MREkAPxT+KzI8Oret+TSKazGq/6gk0uuprdFpkfBXLR5dd4XDoh3NQWp\nyC74VYTDVqJR7IsEaqHtrv01x1qoguU/IRWnzrkGwqXm+Es4W0QZjHisBIRRZ4ywYBG+DtWuskvy\n6/5Mc3dXaUBcm5TnT0c=\n", "msg": { - "state": 5, - "id": 70, - "state_name": state or "ready" + "compose": { + "id": 1, + "state": 4, + } } }} @@ -50,7 +51,7 @@ class ConsumerBaseTest(helpers.ModelsTestCase): class ConsumerTest(ConsumerBaseTest): - @mock.patch("freshmaker.handlers.internal.UpdateDBOnModuleBuild.handle") + @mock.patch("freshmaker.handlers.internal.UpdateDBOnODCSComposeFail.handle") @mock.patch("freshmaker.consumer.get_global_consumer") def test_consumer_processing_message(self, global_consumer, handle): """ @@ -62,16 +63,16 @@ class ConsumerTest(ConsumerBaseTest): global_consumer.return_value = consumer handle.return_value = [freshmaker.events.TestingEvent("ModuleBuilt handled")] - msg = self._module_state_change_msg() + msg = self._compose_state_change_msg() consumer.consume(msg) event = consumer.incoming.get() self.assertEqual(event.msg_id, "ModuleBuilt handled") - @mock.patch("freshmaker.handlers.koji.RebuildImagesOnRPMBodhiUpdate.can_handle") - @mock.patch("freshmaker.handlers.internal.UpdateDBOnModuleBuild.order", + @mock.patch("freshmaker.handlers.koji.RebuildImagesOnODCSComposeDone.can_handle") + @mock.patch("freshmaker.handlers.internal.UpdateDBOnODCSComposeFail.order", new_callable=mock.PropertyMock) - @mock.patch("freshmaker.handlers.internal.UpdateDBOnModuleBuild.can_handle") + @mock.patch("freshmaker.handlers.internal.UpdateDBOnODCSComposeFail.can_handle") @mock.patch("freshmaker.consumer.get_global_consumer") def test_consumer_handlers_order(self, global_consumer, handler1, handler1_order, handler2): @@ -98,14 +99,14 @@ class ConsumerTest(ConsumerBaseTest): handler2.side_effect = mocked_handler2 handler1_order.return_value = 100 if reverse else 0 - msg = self._module_state_change_msg() + msg = self._compose_state_change_msg() consumer.consume(msg) self.assertEqual(order_lst, [2, 1] if reverse else [1, 2]) - @mock.patch("freshmaker.handlers.koji.RebuildImagesOnRPMBodhiUpdate.handle") - @mock.patch("freshmaker.handlers.koji.RebuildImagesOnRPMBodhiUpdate.can_handle") - @mock.patch("freshmaker.handlers.internal.UpdateDBOnModuleBuild.handle") - @mock.patch("freshmaker.handlers.internal.UpdateDBOnModuleBuild.can_handle") + @mock.patch("freshmaker.handlers.koji.RebuildImagesOnODCSComposeDone.handle") + @mock.patch("freshmaker.handlers.koji.RebuildImagesOnODCSComposeDone.can_handle") + @mock.patch("freshmaker.handlers.internal.UpdateDBOnODCSComposeFail.handle") + @mock.patch("freshmaker.handlers.internal.UpdateDBOnODCSComposeFail.can_handle") @mock.patch("freshmaker.consumer.get_global_consumer") def test_consumer_multiple_handlers_called( self, global_consumer, handler1_can_handle, handler1, handler2_can_handle, @@ -115,7 +116,7 @@ class ConsumerTest(ConsumerBaseTest): handler1_can_handle.return_value = True handler2_can_handle.return_value = True - msg = self._module_state_change_msg() + msg = self._compose_state_change_msg() consumer.consume(msg) handler1.assert_called_once() @@ -133,7 +134,7 @@ class ConsumerTest(ConsumerBaseTest): for topic in topics: self.assertIn(mock.call(topic, callback), consumer.hub.subscribe.call_args_list) - @mock.patch("freshmaker.handlers.internal.UpdateDBOnModuleBuild.handle", + @mock.patch("freshmaker.handlers.internal.UpdateDBOnODCSComposeFail.handle", autospec=True) @mock.patch("freshmaker.consumer.get_global_consumer") def test_consumer_mark_event_as_failed_on_exception( @@ -155,7 +156,7 @@ class ConsumerTest(ConsumerBaseTest): handle.side_effect = mocked_handle - msg = self._module_state_change_msg() + msg = self._compose_state_change_msg() consumer.consume(msg) db_event = Event.get(db.session, "msg_id") diff --git a/tests/test_monitor.py b/tests/test_monitor.py index e87169e..b1668ea 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -68,15 +68,16 @@ class ConsumerTest(helpers.ModelsTestCase): def tearDown(self): super(ConsumerTest, self). tearDown() - def _module_state_change_msg(self, state=None): + def _compose_state_change_msg(self, state=None): msg = {'body': { "msg_id": "2017-7afcb214-cf82-4130-92d2-22f45cf59cf7", - "topic": "org.fedoraproject.prod.mbs.module.state.change", + "topic": "org.fedoraproject.prod.odcs.state.change", "signature": "qRZ6oXBpKD/q8BTjBNa4MREkAPxT+KzI8Oret+TSKazGq/6gk0uuprdFpkfBXLR5dd4XDoh3NQWp\nyC74VYTDVqJR7IsEaqHtrv01x1qoguU/IRWnzrkGwqXm+Es4W0QZjHisBIRRZ4ywYBG+DtWuskvy\n6/5Mc3dXaUBcm5TnT0c=\n", "msg": { - "state": 5, - "id": 70, - "state_name": state or "ready" + "compose": { + "id": 1, + "state": 4, + } } }} @@ -90,7 +91,7 @@ class ConsumerTest(helpers.ModelsTestCase): return int(float(v)) return None - @mock.patch("freshmaker.handlers.internal.UpdateDBOnModuleBuild.handle") + @mock.patch("freshmaker.handlers.internal.UpdateDBOnODCSComposeFail.handle") @mock.patch("freshmaker.consumer.get_global_consumer") def test_consumer_processing_message(self, global_consumer, handle): """ @@ -104,7 +105,7 @@ class ConsumerTest(helpers.ModelsTestCase): prev_counter_value = self._get_monitor_value("messaging_rx_processed_ok_total") - msg = self._module_state_change_msg() + msg = self._compose_state_change_msg() consumer.consume(msg) event = consumer.incoming.get() diff --git a/tests/test_pdc.py b/tests/test_pdc.py deleted file mode 100644 index adae26d..0000000 --- a/tests/test_pdc.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2018 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import unittest - -from mock import patch -from freshmaker import conf -from freshmaker.pdc import PDC - - -class TestGetLatestModules(unittest.TestCase): - """Test PDC.get_latest_modules""" - - def mock_is_latest_module(self, module): - fake_results = { - '389-ds': True, - 'apache-commons': True, - 'nodejs': False - } - return fake_results[module['name']] - - @patch('freshmaker.pdc.PDC.get_modules') - @patch('freshmaker.pdc.PDC.is_latest_module') - def test_get_latest_modules(self, is_latest_module, get_modules): - is_latest_module.side_effect = self.mock_is_latest_module - get_modules.side_effect = [ - [ - { - 'uid': '389-ds-1.2-20171120124934', - 'name': '389-ds', - 'stream': '1.2', - 'version': '20171120124934' - }, - { - 'uid': '389-ds-1.2-20171012150041', - 'name': '389-ds', - 'stream': '1.2', - 'version': '20171012150041' - }, - { - 'uid': '389-ds-1.2-20171009091843', - 'name': '389-ds', - 'stream': '1.2', - 'version': '20171009091843' - }, - { - 'uid': 'apache-commons-f27-20171010111836', - 'name': 'apache-commons', - 'stream': 'f27', - 'version': '20171010111836' - }, - { - 'uid': 'nodejs:9:20180213214624:c2c572ec', - 'name': 'nodejs', - 'stream': '9', - 'version': '20180213214624' - }, - { - 'uid': 'nodejs-9-20180205182158', - 'name': 'nodejs', - 'stream': '9', - 'version': '20180205182158' - }, - ], - ] - - pdc = PDC(conf) - modules = pdc.get_latest_modules(build_dep_name='rebuilt module', - build_dep_stream='1.7', - active=True) - modules = sorted(modules, key=lambda m: m['name']) - - # Module nodejs should not be included because the its fake data aims - # to test the nodejs:9:20180213214624:c2c572ec is not latest module. - expected_modules = [ - { - 'uid': '389-ds-1.2-20171120124934', - 'name': '389-ds', - 'stream': '1.2', - 'version': '20171120124934' - }, - { - 'uid': 'apache-commons-f27-20171010111836', - 'name': 'apache-commons', - 'stream': 'f27', - 'version': '20171010111836' - }, - ] - self.assertEqual(expected_modules, modules)