From 73667b267b622dfe77d07ff4d81d4d8ba3b64ee3 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jul 25 2017 06:43:04 +0000 Subject: Store the tree of Docker images to rebuild in Freshmaker DB. --- diff --git a/freshmaker/events.py b/freshmaker/events.py index d056b0c..4aba462 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -259,4 +259,4 @@ class BrewSignRPMEvent(BaseEvent): @property def search_key(self): - return str(self.task_id) + return str(self.nvr) diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index fee9918..0199e5e 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -98,7 +98,8 @@ class BaseHandler(object): namespace=namespace, scratch=conf.koji_container_scratch_build) - def record_build(self, event, name, artifact_type, build_id, dep_on=None): + def record_build(self, event, name, artifact_type, build_id, dep_on=None, + state=None): """ Record build in db. @@ -106,12 +107,18 @@ class BaseHandler(object): :param name: name of the artifact. :param artifact_type: an enum member of ArtifactType. :param build_id: id of the build in build system. - :param def_of: the artifact which this one depends on. + :param dep_on: the artifact which this one depends on. + :param state: the initial state of build. + :return: recorded build. + :rtype: ArtifactBuild. """ ev = models.Event.get_or_create(db.session, event.msg_id, event.search_key, event.__class__) - models.ArtifactBuild.create(db.session, ev, name, artifact_type.name.lower(), build_id, dep_on) + build = models.ArtifactBuild.create( + db.session, ev, name, artifact_type.name.lower(), build_id, + dep_on, state) db.session.commit() + return build def allow_build(self, artifact_type, **kwargs): """ diff --git a/freshmaker/handlers/brew/sign_rpm.py b/freshmaker/handlers/brew/sign_rpm.py index 368f2db..36fd38f 100644 --- a/freshmaker/handlers/brew/sign_rpm.py +++ b/freshmaker/handlers/brew/sign_rpm.py @@ -25,13 +25,15 @@ from itertools import chain from freshmaker import conf from freshmaker import log +from freshmaker import db from freshmaker.events import BrewSignRPMEvent from freshmaker.handlers import BaseHandler from freshmaker.kojiservice import koji_service from freshmaker.lightblue import LightBlue from freshmaker.pulp import Pulp from freshmaker.errata import Errata -from freshmaker.types import ArtifactType +from freshmaker.types import ArtifactType, ArtifactBuildState +import json class BrewSignRPMHanlder(BaseHandler): @@ -66,6 +68,18 @@ class BrewSignRPMHanlder(BaseHandler): log.info('Not find docker images to rebuild.') return [] + self._log_batches(batches) + self._record_batches(batches, event) + + # TODO: build yum repo to contain that signed RPM and start to rebuild + + return [] + + def _log_batches(self, batches): + """ + Logs the information about images to rebuilt using log.info(...). + :param batches list: Output of _find_images_to_rebuild(...). + """ log.info('Found docker images to rebuild in following order:') for i, batch in enumerate(batches): log.info(' Batch %d (%d images):', i, len(batch)) @@ -75,11 +89,33 @@ class BrewSignRPMHanlder(BaseHandler): log.info(' - %s#%s (%s)' % (image["repository"], image["commit"], based_on)) - # TODO: Add batches to database using ArtifactBuild.dep_on + def _record_batches(self, batches, event): + """ + Records the images from batches to database. + :param batches list: Output of _find_images_to_rebuild(...). + """ - # TODO: build yum repo to contain that signed RPM and start to rebuild + # Used as tmp dict with {brew_buil_id: ArtifactBuild, ...} mapping. + builds = {} - return [] + for batch in batches: + for image in batch: + name = image["brew"]["build"] + parent_name = image["parent"]["brew"]["build"] \ + if image["parent"] else None + dep_on = builds[parent_name] if parent_name in builds else None + build = self.record_build( + event, name, ArtifactType.IMAGE, 0, dep_on, + ArtifactBuildState.PLANNED.value) + + build_args = {} + build_args["repository"] = image["repository"] + build_args["commit"] = image["commit"] + build_args["parent"] = parent_name + build.build_args = json.dumps(build_args) + db.session.commit() + + builds[name] = build def _find_images_to_rebuild(self, event): # When get a signed RPM, first step is to find out advisories diff --git a/freshmaker/migrations/versions/8d2e9cd99c54_initial_db.py b/freshmaker/migrations/versions/8d2e9cd99c54_initial_db.py index 945e521..576dd21 100644 --- a/freshmaker/migrations/versions/8d2e9cd99c54_initial_db.py +++ b/freshmaker/migrations/versions/8d2e9cd99c54_initial_db.py @@ -35,6 +35,7 @@ def upgrade(): sa.Column('dep_on_id', sa.Integer(), nullable=True), sa.Column('event_id', sa.Integer(), nullable=True), sa.Column('build_id', sa.Integer(), nullable=True), + sa.Column('build_args', sa.String(), nullable=True), sa.ForeignKeyConstraint(['dep_on_id'], ['artifact_builds.id'], ), sa.ForeignKeyConstraint(['event_id'], ['events.id'], ), sa.PrimaryKeyConstraint('id') diff --git a/freshmaker/models.py b/freshmaker/models.py index 5b3bcf1..3225105 100644 --- a/freshmaker/models.py +++ b/freshmaker/models.py @@ -32,7 +32,7 @@ from freshmaker.types import ArtifactType, ArtifactBuildState from freshmaker.events import ( MBSModuleStateChangeEvent, GitModuleMetadataChangeEvent, GitRPMSpecChangeEvent, TestingEvent, GitDockerfileChangeEvent, - BodhiUpdateCompleteStableEvent, KojiTaskStateChangeEvent) + BodhiUpdateCompleteStableEvent, KojiTaskStateChangeEvent, BrewSignRPMEvent) EVENT_TYPES = { MBSModuleStateChangeEvent: 0, @@ -42,6 +42,7 @@ EVENT_TYPES = { GitDockerfileChangeEvent: 4, BodhiUpdateCompleteStableEvent: 5, KojiTaskStateChangeEvent: 6, + BrewSignRPMEvent: 7, } INVERSE_EVENT_TYPES = {v: k for k, v in EVENT_TYPES.items()} @@ -123,6 +124,9 @@ class ArtifactBuild(FreshmakerBase): # Id of a build in the build system build_id = db.Column(db.Integer) + # Build args in json format. + build_args = db.Column(db.String, nullable=True) + @classmethod def create(cls, session, event, name, type, build_id, dep_on=None, state=None): now = datetime.utcnow() diff --git a/tests/test_brew_sign_rpm_handler.py b/tests/test_brew_sign_rpm_handler.py index dc9688a..4afc2c6 100644 --- a/tests/test_brew_sign_rpm_handler.py +++ b/tests/test_brew_sign_rpm_handler.py @@ -24,12 +24,17 @@ import six import pytest import unittest +import json from mock import patch, MagicMock, PropertyMock from freshmaker.handlers.brew.sign_rpm import BrewSignRPMHanlder from freshmaker.errata import ErrataAdvisory +from freshmaker import db, events +from freshmaker.models import Event +from freshmaker.types import ArtifactBuildState, ArtifactType + @pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') class TestFindBuildSrpmName(unittest.TestCase): @@ -124,3 +129,75 @@ class TestAllowBuild(unittest.TestCase): handler.handle(event) builds_signed.assert_called_once() + + +class TestBatches(unittest.TestCase): + """Test handling of batches""" + + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def _mock_build(self, build, parent=None): + if parent: + parent = {"brew": {"build": parent}} + return {'brew': {'build': build}, 'repository': build + '_repo', + 'commit': build + '_123', 'parent': parent} + + def test_batches_records(self): + """ + Tests that batches are properly recorded in DB. + """ + # Creates following tree: + # shared_parent + # |- child1_parent3 + # |- child1_parent2 + # |- child1_parent1 + # |- child1 + # |- child2_parent2 + # |- child2_parent1 + # |- child2 + batches = [[self._mock_build("shared_parent")], + [self._mock_build("child1_parent3", "shared_parent"), + self._mock_build("child2_parent2", "shared_parent")], + [self._mock_build("child1_parent2", "child1_parent3"), + self._mock_build("child2_parent1", "child2_parent2")], + [self._mock_build("child1_parent1", "child1_parent2"), + self._mock_build("child2", "child2_parent1")], + [self._mock_build("child1", "child1_parent1")]] + + # Flat list of images from batches with brew build id as a key. + images = {} + for batch in batches: + for image in batch: + images[image['brew']['build']] = image + + # Record the batches. + event = events.BrewSignRPMEvent("123", "openssl-1.1.0-1") + handler = BrewSignRPMHanlder() + handler._record_batches(batches, event) + + # Check that the images have proper data in proper db columns. + e = db.session.query(Event).filter(Event.id == 1).one() + for build in e.builds: + self.assertEqual(build.state, ArtifactBuildState.PLANNED.value) + self.assertEqual(build.type, ArtifactType.IMAGE.value) + + image = images[build.name] + if image['parent']: + self.assertEqual(build.dep_on.name, image['parent']['brew']['build']) + else: + self.assertEqual(build.dep_on, None) + + args = json.loads(build.build_args) + self.assertEqual(args["repository"], build.name + "_repo") + self.assertEqual(args["commit"], build.name + "_123") + self.assertEqual(args["parent"], + build.dep_on.name if build.dep_on else None)