From db6e9944dd681956861f90057d10d73480bd7bf0 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Aug 07 2017 07:43:27 +0000 Subject: Prepare yum repo for container rebuild Before rebuilding found containers, yum repo must be created and contain required updated RPMs, which is done in ODCS. freshmaker calls ODCS API to request a new compose and will be used as the repo. Signed-off-by: Chenxiong Qi --- diff --git a/conf/config.py b/conf/config.py index badcc5d..3b53aa3 100644 --- a/conf/config.py +++ b/conf/config.py @@ -127,6 +127,10 @@ class BaseConfiguration(object): # }, # } + # ODCS configs + # URL to ODCS to call APIs + ODCS_SERVER_URL = '' + class DevConfiguration(BaseConfiguration): DEBUG = True diff --git a/freshmaker/config.py b/freshmaker/config.py index eddd185..8277e16 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -228,6 +228,10 @@ class Config(object): 'type': str, 'default': '', 'desc': 'Password to login Pulp.'}, + 'odcs_server_url': { + 'type': str, + 'default': '', + 'desc': 'Server URL to ODCS'}, } def __init__(self, conf_section_obj): diff --git a/freshmaker/handlers/brew/sign_rpm.py b/freshmaker/handlers/brew/sign_rpm.py index 09bab75..e3968d9 100644 --- a/freshmaker/handlers/brew/sign_rpm.py +++ b/freshmaker/handlers/brew/sign_rpm.py @@ -21,6 +21,10 @@ # # Written by Chenxiong Qi +import json +import koji +import time + from itertools import chain from freshmaker import conf @@ -33,7 +37,9 @@ from freshmaker.lightblue import LightBlue from freshmaker.pulp import Pulp from freshmaker.errata import Errata from freshmaker.types import ArtifactType, ArtifactBuildState -import json + +from odcs.client.odcs import ODCS +from odcs.client.odcs import AuthMech class BrewSignRPMHandler(BaseHandler): @@ -71,10 +77,81 @@ class BrewSignRPMHandler(BaseHandler): self._log_batches(batches) self._record_batches(batches, event) - # TODO: build yum repo to contain that signed RPM and start to rebuild + repo_url = self._prepare_yum_repo(event) # noqa return [] + def _prepare_yum_repo(self, event): + """Prepare a yum repo for rebuild + + Run a compose in ODCS to contain required RPMs for rebuilding images + later. + """ + odcs = ODCS(conf.odcs_server_url, auth_mech=AuthMech.Kerberos) + packages = self._get_packages_for_compose(event.nvr) + compose_source = self._get_compose_source(event.nvr) + + if compose_source is None: + log.error('Build %s is not the latest build in its all tags.', + event.nvr) + return + + log.info('Generate new compose for rebuild: ' + 'source: %s, source type: %s, packages: %s', + compose_source, 'tag', packages) + + new_compose = odcs.new_compose(compose_source, + 'tag', + packages=packages) + compose_id = new_compose['id'] + compose_url = '{0}/odcs/{1}/composes/{2}'.format( + odcs.server_url.strip('/'), odcs.api_version, compose_id) + + log.info('Waiting for ODCS to finish the compose: %s', compose_url) + + while True: + time.sleep(1) + + compose = odcs.get_compose(compose_id) + state = compose['state'] + if state == 0: # waiting for generating compose + log.info('Waiting for generating new compose') + elif state == 1: # generating in progress + log.info('ODCS is generating the compose') + elif state == 4: # Failed to generate compose + log.error('ODCS fails to generate compose: %s', compose_url) + log.error('Please consult ODCS to see what is wrong with it') + return + elif state == 2: # Succeed to generate compose + log.info('ODCS has finished to generate compose. Continue to rebuild') + break + else: + log.error('Got unexpected compose state {0} from ODCS.'.format(state)) + return + + log.info('Repo URL containing packages used to rebuild container: %s', + new_compose['result_repo']) + + return new_compose['result_repo'] + + def _get_packages_for_compose(self, nvr): + """Get RPMs of current build NVR""" + with koji_service(conf.koji_profile, log) as session: + rpms = session.get_build_rpms(nvr) + return list(set([rpm['name'] for rpm in rpms])) + + def _get_compose_source(self, nvr): + """Get tag from which to collect packages to compose""" + with koji_service(conf.koji_profile, log) as service: + tag = [tag['name'] for tag in service.session.listTags(nvr) + if tag['name'].endswith('-candidate')][0] + latest_build = service.session.listTagged( + tag, + latest=True, + package=koji.parse_NVR(nvr)['name']) + if latest_build and latest_build[0]['nvr'] == nvr: + return tag + def _log_batches(self, batches): """ Logs the information about images to rebuilt using log.info(...). diff --git a/tests/test_brew_sign_rpm_handler.py b/tests/test_brew_sign_rpm_handler.py index 87cbd2b..e7063a4 100644 --- a/tests/test_brew_sign_rpm_handler.py +++ b/tests/test_brew_sign_rpm_handler.py @@ -26,7 +26,7 @@ import pytest import unittest import json -from mock import patch, MagicMock, PropertyMock +from mock import patch, MagicMock, PropertyMock, Mock from freshmaker.handlers.brew.sign_rpm import BrewSignRPMHandler from freshmaker.errata import ErrataAdvisory @@ -201,3 +201,154 @@ class TestBatches(unittest.TestCase): self.assertEqual(args["commit"], build.name + "_123") self.assertEqual(args["parent"], build.dep_on.name if build.dep_on else None) + + +class TestGetPackagesForCompose(unittest.TestCase): + """Test BrewSignRPMHandler._get_packages_for_compose""" + + @patch('freshmaker.kojiservice.KojiService.get_build_rpms') + def test_get_packages(self, get_build_rpms): + get_build_rpms.return_value = [ + { + 'id': 4672404, + 'arch': 'src', + 'name': 'chkconfig', + 'release': '1.el7_3.1', + 'version': '1.7.2', + 'nvr': 'chkconfig-1.7.2-1.el7_3.1', + }, + { + 'id': 4672405, + 'arch': 'ppc', + 'name': 'chkconfig', + 'release': '1.el7_3.1', + 'version': '1.7.2', + 'nvr': 'chkconfig-1.7.2-1.el7_3.1', + }, + { + 'id': 4672420, + 'arch': 'i686', + 'name': 'chkconfig-debuginfo', + 'release': '1.el7_3.1', + 'version': '1.7.2', + 'nvr': 'chkconfig-debuginfo-1.7.2-1.el7_3.1', + } + ] + + build_nvr = 'chkconfig-1.7.2-1.el7_3.1' + handler = BrewSignRPMHanlder() + packages = handler._get_packages_for_compose(build_nvr) + + get_build_rpms.assert_called_once_with(build_nvr) + + self.assertEqual(set(['chkconfig', 'chkconfig-debuginfo']), + set(packages)) + + +class TestGetComposeSource(unittest.TestCase): + """Test BrewSignRPMHandler._get_compose_source""" + + @patch('freshmaker.kojiservice.KojiService.session', callable=PropertyMock) + def test_get_tag(self, session): + session.listTags.return_value = [ + { + 'id': 10974, + 'name': 'rhscl-3.0-rhel-6-candidate', + }, + { + 'id': 11030, + 'name': 'rhscl-3.0-rhel-6-pending', + }, + { + 'id': 11425, + 'name': 'rhscl-3.0-rhel-6-alpha-1.0-set', + } + ] + session.listTagged.return_value = [ + { + 'build_id': 568228, + 'nvr': 'rh-postgresql96-3.0-9.el6', + } + ] + + handler = BrewSignRPMHanlder() + tag = handler._get_compose_source('rh-postgresql96-3.0-9.el6') + self.assertEqual('rhscl-3.0-rhel-6-candidate', tag) + + @patch('freshmaker.kojiservice.KojiService.session', callable=PropertyMock) + def test_get_None_if_tag_has_new_build(self, session): + session.listTags.return_value = [ + { + 'id': 10974, + 'name': 'rhscl-3.0-rhel-6-candidate', + }, + { + 'id': 11030, + 'name': 'rhscl-3.0-rhel-6-pending', + }, + { + 'id': 11425, + 'name': 'rhscl-3.0-rhel-6-alpha-1.0-set', + } + ] + session.listTagged.return_value = [ + { + 'build_id': 568228, + 'nvr': 'rh-postgresql96-3.0-10.el6', + } + ] + + handler = BrewSignRPMHanlder() + tag = handler._get_compose_source('rh-postgresql96-3.0-9.el6') + self.assertEqual(None, tag) + + +class TestPrepareYumRepo(unittest.TestCase): + """Test BrewSignRPMHandler._prepare_yum_repo""" + + @patch('freshmaker.handlers.brew.sign_rpm.ODCS') + @patch('freshmaker.handlers.brew.sign_rpm.' + 'BrewSignRPMHanlder._get_packages_for_compose') + @patch('freshmaker.handlers.brew.sign_rpm.' + 'BrewSignRPMHanlder._get_compose_source') + @patch('time.sleep') + def test_get_repo_url_when_succeed_to_generate_compose( + self, sleep, _get_compose_source, _get_packages_for_compose, ODCS): + _get_packages_for_compose.return_value = ['httpd', 'httpd-debuginfo'] + _get_compose_source.return_value = 'rhel-7.2-candidate' + ODCS.return_value.new_compose.return_value = { + "id": 3, + "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", + "source": "f26", + "source_type": 1, + "state": 0, + "state_name": "wait", + } + ODCS.return_value.get_compose.return_value = { + "id": 3, + "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", + "source": "f26", + "source_type": 1, + "state": 2, + "state_name": "done", + } + + event = Mock(nvr='httpd-0.1-1.f26') + handler = BrewSignRPMHanlder() + repo_url = handler._prepare_yum_repo(event) + + _get_compose_source.assert_called_once_with(event.nvr) + _get_packages_for_compose.assert_called_once_with(event.nvr) + + # Ensure new_compose is called to request a new compose + ODCS.return_value.new_compose.assert_called_once_with( + 'rhel-7.2-candidate', 'tag', packages=['httpd', 'httpd-debuginfo']) + + # Ensure get_compose is called once in order to get lates state and see + # if it still needs to wait for ODCS + ODCS.return_value.get_compose.assert_called_once_with(3) + + # We should get the right repo URL eventually + self.assertEqual( + 'http://localhost/composes/latest-odcs-3-1/compose/Temporary', + repo_url)