From ecf29feece1f9fab705484937fa6f558db22982d Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jun 01 2018 09:34:55 +0000 Subject: Find images to rebuild based on 'auto_rebuild_tags', not just in 'latest' tag. --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index ebd478d..cf9af57 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -503,6 +503,8 @@ class LightBlue(object): repos = [] for repo_data in response['processed']: + if "auto_rebuild_tags" not in repo_data: + repo_data["auto_rebuild_tags"] = ["latest"] repo = ContainerRepository() repo.update(repo_data) repos.append(repo) @@ -571,7 +573,8 @@ class LightBlue(object): self, published=True, deprecated=False, release_category="Generally Available"): """ - Returns list with names of all matching container repositories. + Returns dict with repository name as key and ContainerRepository as + value. :param bool published: whether to limit queries to published repositories @@ -580,8 +583,9 @@ class LightBlue(object): :param str release_category: filter only repositories with specific release category (options: Deprecated, Generally Available, Beta, Tech Preview) - :rtype: list of str - :return: names of container repositories. + :rtype: dict + :return: Dict with repository name as key and ContainerRepository as + value. """ repo_request = { "objectType": "containerRepository", @@ -590,12 +594,13 @@ class LightBlue(object): }, "projection": [ {"field": "repository", "include": True}, + {"field": "auto_rebuild_tags", "include": True, "recursive": True}, ] } repo_request = self._set_container_repository_filters( repo_request, published, deprecated, release_category) repositories = self.find_container_repositories(repo_request) - return [repository["repository"] for repository in repositories] + return {r["repository"]: r for r in repositories} def _get_default_projection(self, srpm_names=None, include_rpms=True): """ @@ -644,6 +649,10 @@ class LightBlue(object): :param bool published: whether to limit queries to published repositories """ + auto_rebuild_tags = set() + for repo in repositories.values(): + auto_rebuild_tags |= set(repo["auto_rebuild_tags"]) + image_request = { "objectType": "containerImage", "query": { @@ -656,9 +665,11 @@ class LightBlue(object): } for r in content_sets] }, { - "field": "repositories.*.tags.*.name", - "op": "=", - "rvalue": "latest" + "$or": [{ + "field": "repositories.*.tags.*.name", + "op": "=", + "rvalue": tag + } for tag in auto_rebuild_tags] }, { "$or": [{ @@ -672,15 +683,19 @@ class LightBlue(object): "op": "=", "rvalue": "buildfile" }, - { - "field": "repositories.*.published", - "op": "=", - "rvalue": published - }, ] }, "projection": self._get_default_projection(srpm_names=srpm_names) } + + if published is not None: + image_request["query"]["$and"].append( + { + "field": "repositories.*.published", + "op": "=", + "rvalue": published + }) + images = self.find_container_images(image_request) if not images: return images @@ -693,12 +708,15 @@ class LightBlue(object): new_images = [] for image in images: for repository in image["repositories"]: + if repository["repository"] not in repositories: + continue + published_repo = repositories[repository["repository"]] tag_names = [tag["name"] for tag in repository["tags"]] - if (repository["repository"] in repositories and - "latest" in tag_names): - new_images.append(image) + for auto_rebuild_tag in published_repo["auto_rebuild_tags"]: + if auto_rebuild_tag in tag_names: + new_images.append(image) + break images = new_images - return images def find_unpublished_image_for_build(self, build): diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index 0fb1f93..fcea0da 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -390,7 +390,10 @@ class TestContainerImageObject(helpers.FreshmakerTestCase): image.resolve_content_sets(lb) self.assertEqual(image["content_sets"], []) - def test_resolve_content_sets_no_repositories_children_set(self): + @patch('freshmaker.kojiservice.KojiService.get_build') + @patch('freshmaker.kojiservice.KojiService.get_task_request') + def test_resolve_content_sets_no_repositories_children_set( + self, get_task_request, get_build): image = ContainerImage.create({ '_id': '1233829', 'brew': { @@ -473,11 +476,13 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): { "repository": "product/repo1", "content_sets": ["dummy-content-set-1", - "dummy-content-set-2"] + "dummy-content-set-2"], + "auto_rebuild_tags": ["latest", "tag1"], }, { "repository": "product2/repo2", - "content_sets": ["dummy-content-set-1"] + "content_sets": ["dummy-content-set-1"], + "auto_rebuild_tags": ["latest", "tag2"], } ] @@ -556,6 +561,43 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): }, ] + self.fake_images_with_parsed_data_floating_tag = [ + { + 'brew': { + 'completion_date': u'20170421T04:27:51.000-0400', + 'build': 'package-name-3-4-12.10', + 'package': 'package-name-1' + }, + "content_sets": ["dummy-content-set-1", + "dummy-content-set-2"], + 'repositories': [ + {'repository': 'product2/repo2', 'published': True, + 'tags': [{"name": "tag2"}]} + ], + 'parsed_data': { + 'files': [ + { + 'key': 'buildfile', + 'content_url': 'http://git.repo.com/cgit/rpms/repo-1/plain/Dockerfile?id=commit_hash1', + 'filename': u'Dockerfile' + } + ], + }, + 'rpm_manifest': [{ + 'rpms': [ + { + "srpm_name": "openssl", + "srpm_nevra": "openssl-0:1.2.3-1.src" + }, + { + "srpm_name": "tespackage", + "srpm_nevra": "testpackage-10:1.2.3-1.src" + } + ] + }] + }, + ] + self.fake_container_images = [ ContainerImage.create(data) for data in self.fake_images_with_parsed_data] @@ -690,6 +732,7 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): self.assertEqual('20160927T11:14:56.420-0400', repo['creationDate']) self.assertEqual(0, repo['metrics']['pulls_in_last_30_days']) self.assertEqual('20170223T08:28:40.913-0500', repo['metrics']['last_update_date']) + self.assertEqual(["latest"], repo["auto_rebuild_tags"]) @patch('freshmaker.lightblue.requests.post') def test_raise_error_if_request_data_is_incorrect(self, post): @@ -758,13 +801,14 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): }, "projection": [ {"field": "repository", "include": True}, + {"field": "auto_rebuild_tags", "include": True, "recursive": True}, ] } cont_repos.assert_called_with(expected_repo_request) - expected_ret = [ - repo["repository"] for repo in - self.fake_repositories_with_content_sets] + expected_ret = { + repo["repository"]: repo for repo in + self.fake_repositories_with_content_sets} self.assertEqual(ret, expected_ret) @patch('freshmaker.lightblue.LightBlue.find_container_images') @@ -776,9 +820,9 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): lb = LightBlue(server_url=self.fake_server_url, cert=self.fake_cert_file, private_key=self.fake_private_key) - repositories = [ - repo["repository"] for repo in - self.fake_repositories_with_content_sets] + repositories = { + repo["repository"]: repo for repo in + self.fake_repositories_with_content_sets} cont_images.return_value = self.fake_images_with_parsed_data ret = lb.find_images_with_included_srpms( ["content-set-1", "content-set-2"], ["openssl"], repositories) @@ -802,9 +846,23 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): ], }, { - "field": "repositories.*.tags.*.name", - "op": "=", - "rvalue": "latest" + "$or": [ + { + "field": "repositories.*.tags.*.name", + "op": "=", + "rvalue": "tag1" + }, + { + "field": "repositories.*.tags.*.name", + "op": "=", + "rvalue": "tag2" + }, + { + "field": "repositories.*.tags.*.name", + "op": "=", + "rvalue": "latest" + }, + ], }, { "$or": [ @@ -837,6 +895,28 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): # in repository "product/repo1". self.assertEqual(ret, [cont_images.return_value[1]]) + @patch('freshmaker.lightblue.LightBlue.find_container_images') + @patch('os.path.exists') + def test_images_with_included_srpm_floating_tag( + self, exists, cont_images): + + exists.return_value = True + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + repositories = { + repo["repository"]: repo for repo in + self.fake_repositories_with_content_sets} + cont_images.return_value = ( + self.fake_images_with_parsed_data + + self.fake_images_with_parsed_data_floating_tag) + ret = lb.find_images_with_included_srpms( + ["content-set-1", "content-set-2"], ["openssl"], repositories) + + self.assertEqual( + [image["brew"]["build"] for image in ret], + ['package-name-2-4-12.10', 'package-name-3-4-12.10']) + def _filter_fnc(self, image): return image["brew"]["build"].startswith("filtered_")