From 115c2810fcd4d81ffeaec4701e2279bac7e47ebf Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jan 21 2019 11:25:33 +0000 Subject: Initial code to send rebuild requests to Bob on Errata advisory state change. This adds new handlers.bob.RebuildImagesOnImageAdvisoryChange handler which handles the ErrataAdvisoryStateChangedEvent. This handler uses Errata Tool and Pulp to get the list of Docker repositories in which the images attached to advisory will end up (ended up). It then requests Bob to rebuild images depending on them. This also adds new pulp_docker_* config variables, because Docker images are stored in different Pulp instance than RPMs. --- diff --git a/conf/config.py b/conf/config.py index d3ccac1..39ca037 100644 --- a/conf/config.py +++ b/conf/config.py @@ -320,6 +320,7 @@ class TestConfiguration(BaseConfiguration): } KRB_AUTH_CCACHE_FILE = "freshmaker_cc_$pid_$tid" + ERRATA_TOOL_SERVER_URL = "http://localhost/" # fake URL just for tests. class ProdConfiguration(BaseConfiguration): diff --git a/freshmaker/config.py b/freshmaker/config.py index 398f959..9bc4de1 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -294,6 +294,26 @@ class Config(object): 'type': str, 'default': '', 'desc': 'Password to login Pulp.'}, + 'pulp_docker_server_url': { + 'type': str, + 'default': '', + 'desc': 'Server URL of Pulp Docker.'}, + 'pulp_docker_username': { + 'type': str, + 'default': '', + 'desc': 'Username to login Pulp Docker.'}, + 'pulp_docker_password': { + 'type': str, + 'default': '', + 'desc': 'Password to login Pulp Docker.'}, + 'bob_server_url': { + 'type': str, + 'default': '', + 'desc': 'Server URL of Bob container images rebuild service.'}, + 'bob_auth_token': { + 'type': str, + 'default': '', + 'desc': 'Auth token for Bob container images rebuild service.'}, 'odcs_server_url': { 'type': str, 'default': '', diff --git a/freshmaker/errata.py b/freshmaker/errata.py index 3b30dfa..64ec90b 100644 --- a/freshmaker/errata.py +++ b/freshmaker/errata.py @@ -21,10 +21,12 @@ # # Written by Jan Kaluza +import xmlrpclib import os import requests import dogpile.cache from requests_kerberos import HTTPKerberosAuth +from kobo.xmlrpc import SafeCookieTransport from freshmaker.events import ( BrewSignRPMEvent, ErrataBaseEvent, @@ -117,6 +119,10 @@ class Errata(object): else: self.server_url = conf.errata_tool_server_url.rstrip('/') + xmlrpc_url = self.server_url + '/errata/xmlrpc.cgi' + self.xmlrpc = xmlrpclib.ServerProxy( + xmlrpc_url, transport=SafeCookieTransport()) + def _errata_authorized_get(self, *args, **kwargs): r = requests.get( *args, @@ -175,6 +181,45 @@ class Errata(object): return advisories + def get_docker_repo_tags(self, errata_id): + """ + Get ET repo/tag configuration using XML-RPC call + get_advisory_cdn_docker_file_list + :param int errata_id: Errata advisory ID. + :rtype: dict + :return: Dict of advisory builds with repo and tag config: + { + 'build_NVR': { + 'cdn_repo1': [ + 'tag1', + 'tag2' + ], + ... + }, + ... + } + """ + try: + response = self.xmlrpc.get_advisory_cdn_docker_file_list( + errata_id) + except Exception: + log.exception("Canot call XMLRPC get_advisory_cdn_docker_file_list call.") + return None + if response is None: + log.warning("The get_advisory_cdn_docker_file_list XMLRPC call " + "returned None.") + return None + + repo_tags = dict() + for build_nvr in response: + if build_nvr not in repo_tags: + repo_tags[build_nvr] = dict() + repos = response[build_nvr]['docker']['target']['repos'] + for repo in repos: + tags = repos[repo]['tags'] + repo_tags[build_nvr][repo] = tags + return repo_tags + def advisories_from_event(self, event): """ Returns list of ErrataAdvisory instances associated with diff --git a/freshmaker/handlers/bob/__init__.py b/freshmaker/handlers/bob/__init__.py new file mode 100644 index 0000000..89f4570 --- /dev/null +++ b/freshmaker/handlers/bob/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .rebuild_images_on_image_advisory_change import RebuildImagesOnImageAdvisoryChange # noqa diff --git a/freshmaker/handlers/bob/rebuild_images_on_image_advisory_change.py b/freshmaker/handlers/bob/rebuild_images_on_image_advisory_change.py new file mode 100644 index 0000000..adb9ce5 --- /dev/null +++ b/freshmaker/handlers/bob/rebuild_images_on_image_advisory_change.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +import requests + +from freshmaker import conf, db +from freshmaker.models import Event +from freshmaker.errata import Errata +from freshmaker.pulp import Pulp +from freshmaker.events import ErrataAdvisoryStateChangedEvent +from freshmaker.handlers import ContainerBuildHandler, fail_event_on_handler_exception +from freshmaker.types import EventState + + +class RebuildImagesOnImageAdvisoryChange(ContainerBuildHandler): + name = 'RebuildImagesOnImageAdvisoryChange' + + def can_handle(self, event): + return isinstance(event, ErrataAdvisoryStateChangedEvent) + + @fail_event_on_handler_exception + def handle(self, event): + if event.dry_run: + self.force_dry_run() + + db_event = Event.get_or_create_from_event(db.session, event) + self.set_context(db_event) + + # Check if we are allowed to build this advisory. + if not event.is_allowed(self): + msg = ("Errata advisory {0} is not allowed by internal policy " + "to trigger Bob rebuilds.".format(event.advisory.errata_id)) + db_event.transition(EventState.SKIPPED, msg) + db.session.commit() + self.log_info(msg) + return [] + + self.rebuild_images_depending_on_advisory( + db_event, event.advisory.errata_id) + + def rebuild_images_depending_on_advisory(self, db_event, errata_id): + """ + Submits requests to Bob to rebuild the images depending on the + images updated in the advisory with ID `errata_id`. + """ + # Get the list of CDN repository names for each build in the advisory + # as well as the name of tags used for the images. + errata = Errata() + repo_tags = errata.get_docker_repo_tags(errata_id) + if not repo_tags: + msg = "No CDN repo found for advisory %r" % errata_id + self.log_info(msg) + db_event.transition(EventState.FAILED, msg) + db.session.commit() + return + + # Use the Pulp to get the Docker repository name from the CDN repository + # name and store it into `docker_repos` dict. + pulp = Pulp(conf.pulp_docker_server_url, conf.pulp_docker_username, + conf.pulp_docker_password) + # {docker_repository_name: [list, of, docker, tags], ...} + docker_repos = {} + for per_build_repo_tags in repo_tags.values(): + for cdn_repo, docker_repo_tags in per_build_repo_tags.items(): + docker_repo = pulp.get_docker_repository_name(cdn_repo) + if not docker_repo: + self.log_error("No Docker repo found for CDN repo %r", cdn_repo) + continue + docker_repos[docker_repo] = docker_repo_tags + + self.log_info("Found following Docker repositories updated by the advisory: %r", + docker_repos.keys()) + + # Submit rebuild request to Bob :). + for repo_name in docker_repos.keys(): + self.log_info("Requesting Bob rebuild of %s", repo_name) + bob_url = "%s/update_children/%s" % ( + conf.bob_server_url.rstrip('/'), repo_name) + headers = {"Authorization": "Bearer %s" % conf.bob_auth_token} + if self.dry_run: + self.log_info("DRY RUN: Skipping request to Bob.") + continue + + r = requests.get(bob_url, headers=headers) + r.raise_for_status() + # TODO: Once the Bob API is clear here, we can handle the response, + # but for now just log it. This should also be changed to log_debug + # once we are in production, but for now log_info makes debugging + # this new code easier. + self.log_info("Response: %r", r.json()) + + db_event.transition(EventState.COMPLETE) + db.session.commit() diff --git a/freshmaker/pulp.py b/freshmaker/pulp.py index 027bdfb..1d89b05 100644 --- a/freshmaker/pulp.py +++ b/freshmaker/pulp.py @@ -42,6 +42,14 @@ class Pulp(object): r.raise_for_status() return r.json() + def _rest_get(self, endpoint, **kwargs): + r = requests.get( + '{0}{1}'.format(self.rest_api_root, endpoint.lstrip('/')), + params=kwargs, + auth=(self.username, self.password)) + r.raise_for_status() + return r.json() + def get_content_set_by_repo_ids(self, repo_ids): """Get content_sets by repository IDs @@ -60,3 +68,22 @@ class Pulp(object): repos = self._rest_post('repositories/search/', json.dumps(query_data)) return [repo['notes']['content_set'] for repo in repos if 'content_set' in repo['notes']] + + def get_docker_repository_name(self, cdn_repo): + """ + Getting docker repository name from pulp using cdn repo name. + + :param str cdn_repo: The CDN repo name from Errata Tool. + :rtype: str + :return: Docker repository name. + """ + response = self._rest_get( + 'repositories/%s/' % cdn_repo, distributors=True) + + docker_repository_name = None + for distributor in response['distributors']: + if distributor['distributor_type_id'] == 'docker_distributor_web': + docker_repository_name = \ + distributor['config']['repo-registry-id'] + break + return docker_repository_name diff --git a/tests/handlers/bob/__init__.py b/tests/handlers/bob/__init__.py new file mode 100644 index 0000000..043e3c6 --- /dev/null +++ b/tests/handlers/bob/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza diff --git a/tests/handlers/bob/test_rebuild_images_on_image_advisory_change.py b/tests/handlers/bob/test_rebuild_images_on_image_advisory_change.py new file mode 100644 index 0000000..cfab08b --- /dev/null +++ b/tests/handlers/bob/test_rebuild_images_on_image_advisory_change.py @@ -0,0 +1,120 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from mock import patch, MagicMock, call + +import freshmaker +from freshmaker.errata import ErrataAdvisory +from freshmaker.events import ErrataAdvisoryStateChangedEvent +from freshmaker.handlers.bob import RebuildImagesOnImageAdvisoryChange +from tests import helpers + + +class RebuildImagesOnImageAdvisoryChangeTest(helpers.ModelsTestCase): + + def setUp(self): + super(RebuildImagesOnImageAdvisoryChangeTest, self).setUp() + + self.event = ErrataAdvisoryStateChangedEvent( + "123", + ErrataAdvisory(123, "RHBA-2017", "SHIPPED_LIVE", [], + security_impact="", + product_short_name="product")) + self.handler = RebuildImagesOnImageAdvisoryChange() + self.db_event = MagicMock() + + @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ + 'RebuildImagesOnImageAdvisoryChange': { + "image": {"advisory_state": "SHIPPED_LIVE"} + } + }) + @patch("freshmaker.handlers.bob.RebuildImagesOnImageAdvisoryChange." + "rebuild_images_depending_on_advisory") + def test_handler_allowed(self, rebuild_images): + self.event.advisory.state = "NEW_FILES" + self.handler.handle(self.event) + rebuild_images.assert_not_called() + + self.event.advisory.state = "SHIPPED_LIVE" + self.handler.handle(self.event) + rebuild_images.assert_called_once() + + @patch("freshmaker.errata.Errata.get_docker_repo_tags") + @patch("freshmaker.pulp.Pulp.get_docker_repository_name") + @patch("freshmaker.handlers.bob." + "rebuild_images_on_image_advisory_change.requests.get") + @patch.object(freshmaker.conf, 'bob_auth_token', new="x") + @patch.object(freshmaker.conf, 'bob_server_url', new="http://localhost/") + def test_rebuild_images_depending_on_advisory( + self, requests_get, get_docker_repository_name, + get_docker_repo_tags): + get_docker_repo_tags.return_value = { + 'foo-container-1-1': {'foo-526': ['5.26', 'latest']}, + 'bar-container-1-1': {'bar-526': ['5.26', 'latest']}} + get_docker_repository_name.side_effect = [ + "scl/foo-526", "scl/bar-526"] + self.handler.rebuild_images_depending_on_advisory(self.db_event, 123) + + get_docker_repo_tags.assert_called_once_with(123) + get_docker_repository_name.assert_has_calls([ + call("bar-526"), call("foo-526")]) + requests_get.assert_any_call( + 'http://localhost/update_children/scl/foo-526', + headers={'Authorization': 'Bearer x'}) + requests_get.assert_any_call( + 'http://localhost/update_children/scl/bar-526', + headers={'Authorization': 'Bearer x'}) + + @patch("freshmaker.errata.Errata.get_docker_repo_tags") + @patch("freshmaker.pulp.Pulp.get_docker_repository_name") + @patch("freshmaker.handlers.bob." + "rebuild_images_on_image_advisory_change.requests.get") + @patch.object(freshmaker.conf, 'bob_auth_token', new="x") + @patch.object(freshmaker.conf, 'bob_server_url', new="http://localhost/") + def test_rebuild_images_depending_on_advisory_unknown_advisory( + self, requests_get, get_docker_repository_name, + get_docker_repo_tags): + get_docker_repo_tags.return_value = None + self.handler.rebuild_images_depending_on_advisory(self.db_event, 123) + + get_docker_repo_tags.assert_called_once_with(123) + get_docker_repository_name.assert_not_called() + requests_get.assert_not_called() + + @patch("freshmaker.errata.Errata.get_docker_repo_tags") + @patch("freshmaker.pulp.Pulp.get_docker_repository_name") + @patch("freshmaker.handlers.bob." + "rebuild_images_on_image_advisory_change.requests.get") + @patch.object(freshmaker.conf, 'bob_auth_token', new="x") + @patch.object(freshmaker.conf, 'bob_server_url', new="http://localhost/") + def test_rebuild_images_depending_on_advisory_dry_run( + self, requests_get, get_docker_repository_name, + get_docker_repo_tags): + get_docker_repo_tags.return_value = { + 'foo-container-1-1': {'foo-526': ['5.26', 'latest']}} + get_docker_repository_name.return_value = "scl/foo-526" + self.handler.force_dry_run() + self.handler.rebuild_images_depending_on_advisory(self.db_event, 123) + + get_docker_repo_tags.assert_called_once_with(123) + get_docker_repository_name.assert_called_once_with("foo-526") + requests_get.assert_not_called() diff --git a/tests/test_errata.py b/tests/test_errata.py index f63bb59..57c1424 100644 --- a/tests/test_errata.py +++ b/tests/test_errata.py @@ -321,3 +321,34 @@ class TestErrata(helpers.FreshmakerTestCase): MockedErrataAPI(errata_rest_get, errata_http_get) ret = self.errata.get_builds(28484, "RHEL-7") self.assertEqual(ret, set(['libntirpc-1.4.3-4.el7rhgs'])) + + def test_get_docker_repo_tags(self): + with patch.object(self.errata, "xmlrpc") as xmlrpc: + xmlrpc.get_advisory_cdn_docker_file_list.return_value = { + 'foo-container-1-1': { + 'docker': { + 'target': { + 'repos': { + 'foo-526': {'tags': ['5.26', 'latest']} + } + } + } + } + } + repo_tags = self.errata.get_docker_repo_tags(28484) + + expected = {'foo-container-1-1': {'foo-526': ['5.26', 'latest']}} + self.assertEqual(repo_tags, expected) + + def test_get_docker_repo_tags_xmlrpc_exception(self): + with patch.object(self.errata, "xmlrpc") as xmlrpc: + xmlrpc.get_advisory_cdn_docker_file_list.side_effect = ValueError( + "Expected XMLRPC test exception") + repo_tags = self.errata.get_docker_repo_tags(28484) + self.assertEqual(repo_tags, None) + + def test_get_docker_repo_tags_xmlrpc_non_returned(self): + with patch.object(self.errata, "xmlrpc") as xmlrpc: + xmlrpc.get_advisory_cdn_docker_file_list.return_value = None + repo_tags = self.errata.get_docker_repo_tags(28484) + self.assertEqual(repo_tags, None) diff --git a/tests/test_pulp.py b/tests/test_pulp.py index 0332c45..205d911 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -158,3 +158,25 @@ class TestPulp(helpers.FreshmakerTestCase): self.assertEqual(['rhel-7-workstation-rpms', 'rhel-7-desktop-rpms'], content_sets) + + @patch('freshmaker.pulp.requests.get') + def test_get_docker_repository_name(self, get): + get.return_value.json.return_value = { + 'display_name': 'foo-526', + 'description': 'Foo', + 'distributors': [ + {'repo_id': 'foo-526', + 'distributor_type_id': 'docker_distributor_web', + 'config': {'repo-registry-id': 'scl/foo-526'}} + ] + } + + pulp = Pulp(self.server_url, username=self.username, password=self.password) + repo_name = pulp.get_docker_repository_name("foo-526") + + get.assert_called_once_with( + '{}pulp/api/v2/repositories/foo-526/'.format(self.server_url), + params={"distributors": True}, + auth=(self.username, self.password)) + + self.assertEqual(repo_name, "scl/foo-526")