From 643ba670ee58d1053ed46c20a0ba34f954ba1adb Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jul 20 2018 13:18:48 +0000 Subject: [PATCH 1/2] Cache ODCS pulp composes with the same content_sets in record_batches. --- diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index 889169f..8a071d6 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -503,6 +503,10 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): # Used as tmp dict with {brew_build_nvr: ArtifactBuild, ...} mapping. builds = builds or {} + # Cache for ODCS pulp composes. Key is white-spaced, sorted, list + # of content_sets. Value is ODCS compose JSON response. + odcs_cache = {} + for batch in batches: for image in batch: nvr = image["brew"]["build"] @@ -560,12 +564,30 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): if state != ArtifactBuildState.FAILED.value: # Store odcs pulp compose to build if image["generate_pulp_repos"]: - compose = self._prepare_pulp_repo( - build, image["content_sets"]) + # Check if the compose for these content_sets is + # already cached and use it in this case. + cache_key = " ".join(sorted(image["content_sets"])) + if cache_key in odcs_cache: + using_cached_compose = True + compose = odcs_cache[cache_key] + else: + using_cached_compose = False + compose = self._prepare_pulp_repo( + build, image["content_sets"]) + if build.state != ArtifactBuildState.FAILED.value: - db_compose = Compose(odcs_compose_id=compose['id']) - db.session.add(db_compose) - db.session.commit() + if using_cached_compose: + # In case we are using cached compose, get + # the DB representation of this compose. + db_compose = db.session.query(Compose).filter_by( + odcs_compose_id=compose["id"]).first() + else: + # Otherwise cache the compose and create record + # in the DB. + odcs_cache[cache_key] = compose + db_compose = Compose(odcs_compose_id=compose['id']) + db.session.add(db_compose) + db.session.commit() build.add_composes(db.session, [db_compose]) else: db.session.commit() diff --git a/tests/test_errata_advisory_state_changed.py b/tests/test_errata_advisory_state_changed.py index 66814d1..be81b71 100644 --- a/tests/test_errata_advisory_state_changed.py +++ b/tests/test_errata_advisory_state_changed.py @@ -805,7 +805,7 @@ class TestRecordBatchesImages(helpers.ModelsTestCase): side_effect=[{'id': 100}, {'id': 200}]) self.patcher.patch_dict( - 'freshmaker.models.EVENT_TYPES', {self.mock_event.__class__: -1}) + 'freshmaker.models.EVENT_TYPES', {self.mock_event.__class__: 0}) def tearDown(self): super(TestRecordBatchesImages, self).tearDown() @@ -1028,6 +1028,95 @@ class TestRecordBatchesImages(helpers.ModelsTestCase): self.mock_request_boot_iso_compose.assert_called_once_with( batches[0][0]) + def test_pulp_compose_generated_just_once(self): + batches = [ + [ContainerImage({ + "brew": { + "completion_date": "20170420T17:05:37.000-0400", + "build": "rhel-server-docker-7.3-82", + "package": "rhel-server-docker" + }, + 'parsed_data': { + 'layers': [ + 'sha512:12345678980', + 'sha512:10987654321' + ] + }, + "parent": None, + "content_sets": ["content-set-1"], + "repository": "repo-1", + "commit": "123456789", + "target": "target-candidate", + "git_branch": "rhel-7", + "error": None, + "arches": "x86_64", + "generate_pulp_repos": True, + })], + [ContainerImage({ + "brew": { + "build": "rh-dotnetcore10-docker-1.0-16", + "package": "rh-dotnetcore10-docker", + "completion_date": "20170511T10:06:09.000-0400" + }, + 'parsed_data': { + 'layers': [ + 'sha512:2345af2e293', + 'sha512:12345678980', + 'sha512:10987654321' + ] + }, + "parent": ContainerImage({ + "brew": { + "completion_date": "20170420T17:05:37.000-0400", + "build": "rhel-server-docker-7.3-82", + "package": "rhel-server-docker" + }, + 'parsed_data': { + 'layers': [ + 'sha512:12345678980', + 'sha512:10987654321' + ] + }, + "parent": None, + "content_sets": ["content-set-1"], + "repository": "repo-1", + "commit": "123456789", + "target": "target-candidate", + "git_branch": "rhel-7", + "error": None + }), + "content_sets": ["content-set-1"], + "repository": "repo-1", + "commit": "987654321", + "target": "target-candidate", + "git_branch": "rhel-7", + "error": None, + "arches": "x86_64", + "generate_pulp_repos": True, + })] + ] + + handler = ErrataAdvisoryRPMsSignedHandler() + handler._record_batches(batches, self.mock_event) + + query = db.session.query(ArtifactBuild) + parent_build = query.filter( + ArtifactBuild.original_nvr == 'rhel-server-docker-7.3-82' + ).first() + self.assertEqual(1, len(parent_build.composes)) + compose_ids = sorted([rel.compose.odcs_compose_id + for rel in parent_build.composes]) + self.assertEqual([1], compose_ids) + + child_build = query.filter( + ArtifactBuild.original_nvr == 'rh-dotnetcore10-docker-1.0-16' + ).first() + self.assertEqual(1, len(child_build.composes)) + + self.mock_prepare_pulp_repo.assert_has_calls([ + call(parent_build, ["content-set-1"]) + ]) + def test_no_parent(self): batches = [ [ContainerImage({ From 35ae054f15a9d782dc7da474fb600df921b9ae3c Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jul 23 2018 12:10:44 +0000 Subject: [PATCH 2/2] Cache the Compose database object instead of ODCS compose dict. --- diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index 8a071d6..87da215 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -504,7 +504,7 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): builds = builds or {} # Cache for ODCS pulp composes. Key is white-spaced, sorted, list - # of content_sets. Value is ODCS compose JSON response. + # of content_sets. Value is Compose database object. odcs_cache = {} for batch in batches: @@ -568,28 +568,21 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): # already cached and use it in this case. cache_key = " ".join(sorted(image["content_sets"])) if cache_key in odcs_cache: - using_cached_compose = True - compose = odcs_cache[cache_key] + db_compose = odcs_cache[cache_key] else: - using_cached_compose = False compose = self._prepare_pulp_repo( build, image["content_sets"]) - if build.state != ArtifactBuildState.FAILED.value: - if using_cached_compose: - # In case we are using cached compose, get - # the DB representation of this compose. - db_compose = db.session.query(Compose).filter_by( - odcs_compose_id=compose["id"]).first() - else: - # Otherwise cache the compose and create record - # in the DB. - odcs_cache[cache_key] = compose + if build.state != ArtifactBuildState.FAILED.value: db_compose = Compose(odcs_compose_id=compose['id']) db.session.add(db_compose) db.session.commit() + odcs_cache[cache_key] = db_compose + else: + db_compose = None + db.session.commit() + if db_compose: build.add_composes(db.session, [db_compose]) - else: db.session.commit() # TODO: uncomment following code after boot.iso compose is