From eb60dcbf889580dbc9b0a41a24e8df16770bf646 Mon Sep 17 00:00:00 2001 From: Stanislav Ochotnicky Date: Jun 01 2017 07:47:28 +0000 Subject: [PATCH 1/3] Add method to query repositories with specific content sets containerRepository entities have "content_sets" array of strings. This commit adds a method to query all repositories which contain at least one of the content sets from the argument. By default we ignore repositories which are unpublished, deprecated or with a category other than "Generally Available" --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index abda908..03cfe70 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -204,3 +204,56 @@ class LightBlue(object): image.update(image_data) images.append(image) return images + + def find_repositories_with_content_sets(self, + content_sets, + published=True, + deprecated=False, + release_category="Generally Available"): + """Query lightblue and find containerRepositories which have content + from at least one of the content_sets. By default ignore unpublished, + deprecated repos or non-GA repositories + + :param list content_sets: list of strings (content sets) to consider + when looking for the packages + :param bool published: whether to limit queries to published + repositories + :param bool deprecated: set to True to limit results to deprecated + repositories + :param str release_category: filter only repositories with specific + release category (options: Deprecated, Generally Available, Beta, Tech Preview) + """ + repo_request = { + "objectType": "containerRepository", + "query": { + "$and": [ + { + "$or": [{ + "field": "content_sets.*", + "op": "=", + "rvalue": c + } for c in content_sets] + }, + { + "field": "published", + "op": "=", + "rvalue": published + }, + { + "field": "deprecated", + "op": "=", + "rvalue": deprecated + }, + { + "field": "release_categories.*", + "op": "=", + "rvalue": release_category + } + ] + }, + "projection": [ + {"field": "repository", "include": True}, + {"field": "content_sets", "include": True, "recursive": True} + ] + } + return self.find_container_repositories(repo_request) diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index b60bc67..fbb0bc1 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -110,6 +110,77 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): self.fake_server_url = 'lightblue.localhost' self.fake_cert_file = 'path/to/cert' self.fake_private_key = 'path/to/private-key' + self.fake_repositories_with_content_sets = [ + { + "repository": "product/repo1", + "content_sets": ["dummy-content-set-1", + "dummy-content-set-2"] + }, + { + "repository": "product2/repo2", + "content_sets": ["dummy-content-set-1"] + } + ] + + self.fake_images_with_parsed_data = [ + { + 'brew': { + 'completion_date': u'20170421T04:27:51.000-0400', + 'build': 'package-name-1-4-12.10', + 'package': 'package-name-1' + }, + 'parsed_data': { + 'files': [ + { + 'key': 'buildfile', + 'content_url': 'http://git.repo.com/cgit/rpms/repo-1/plain/Dockerfile?id=commit_hash1', + 'filename': u'Dockerfile' + } + ], + 'rpm_manifest': [ + { + "srpm_name": "openssl", + "srpm_nevra": "openssl-0:1.2.3-1.src" + }, + { + "srpm_name": "tespackage", + "srpm_nevra": "testpackage-10:1.2.3-1.src" + } + ] + } + }, + { + 'brew': { + 'completion_date': u'20170421T04:27:51.000-0400', + 'build': 'package-name-2-4-12.10', + 'package': 'package-name-2' + }, + 'parsed_data': { + 'files': [ + { + 'key': 'buildfile', + 'content_url': 'http://git.repo.com/cgit/ns/repo-2/plain/Dockerfile?id=commit_hash2', + 'filename': 'Dockerfile' + }, + { + 'key': 'bogusfile', + 'content_url': 'bogus_test_url', + 'filename': 'bogus.file' + } + ], + 'rpm_manifest': [ + { + "srpm_name": "openssl", + "srpm_nevra": "openssl-1:1.2.3-1.src" + }, + { + "srpm_name": "tespackage2", + "srpm_nevra": "testpackage2-10:1.2.3-1.src" + } + ] + } + }, + ] @patch('freshmaker.lightblue.requests.post') def test_find_container_images(self, post): @@ -266,6 +337,56 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): self.assertRaises(LightBlueRequestFailure, lb._make_request, 'find/containerRepository/', fake_request) + @patch('freshmaker.lightblue.LightBlue.find_container_repositories') + @patch('os.path.exists') + def test_find_repositories_with_content_sets(self, exists, + cont_repos): + exists.return_value = True + queried_content_set = "rhel-7-server-rpms" + cont_repos.return_value = self.fake_repositories_with_content_sets + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + ret = lb.find_repositories_with_content_sets([queried_content_set]) + expected_repo_request = { + "objectType": "containerRepository", + "query": { + "$and": [ + { + "$or": [ + { + "field": "content_sets.*", + "op": "=", + "rvalue": queried_content_set + } + ], + }, + { + "field": "published", + "op": "=", + "rvalue": True + }, + { + "field": "deprecated", + "op": "=", + "rvalue": False + }, + { + "field": "release_categories.*", + "op": "=", + "rvalue": "Generally Available" + } + ] + }, + "projection": [ + {"field": "repository", "include": True}, + {"field": "content_sets", "include": True, "recursive": True} + + ] + } + cont_repos.assert_called_with(expected_repo_request) + self.assertEqual(ret, cont_repos.return_value) + class TestEntityVersion(unittest.TestCase): """Test case for ensuring correct entity version in request""" From d759bb3792909ad567c36fa09fc702ddabef918c Mon Sep 17 00:00:00 2001 From: Stanislav Ochotnicky Date: Jun 01 2017 07:49:47 +0000 Subject: [PATCH 2/3] Add method for querying images with specific srpm_name This commit adds method which given set of repositories and a srpm_name returns image objects from those repositories which have rpms coming from source rpm with given name. This makes use of rpm_manifest entities in containerImages. By default unpublished or deprecated images are ignored. We also limit the results to images with latest tag to ignore older images. In the future we should improve this to include any floating tags not just latest. --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 03cfe70..d667e19 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -257,3 +257,58 @@ class LightBlue(object): ] } return self.find_container_repositories(repo_request) + + + def find_images_with_included_srpm(self, repositories, srpm_name, + published=True): + + """Query lightblue and find containerImages in given + containerRepositories. By default limit only to images which have been + published to at least one repository and images which have latest tag. + + :param dict repositories: dictionary with repository names to look inside + :param str srpm_name: srpm_name (source rpm name) to look for + :param bool published: whether to limit queries to images with at least + one published repository + """ + image_request = { + "objectType": "containerImage", + "query": { + "$and": [ + { + "$or": [{ + "field": "repositories.*.repository", + "op": "=", + "rvalue": r['repository'] + } for r in repositories] + }, + { + "field": "repositories.*.published", + "op": "=", + "rvalue": published + }, + { + "field": "repositories.*.tags.*.name", + "op": "=", + "rvalue": "latest" + }, + { + "field": "parsed_data.rpm_manifest.*.srpm_name", + "op": "=", + "rvalue": srpm_name + }, + { + "field": "parsed_data.files.*.key", + "op": "=", + "rvalue": "buildfile" + } + ] + }, + "projection": [ + {"field": "brew", "include": True, "recursive": True}, + {"field": "parsed_data.files", "include": True, "recursive": True}, + {"field": "parsed_data.rpm_manifest.*.srpm_nevra", "include": True, "recursive": True}, + {"field": "parsed_data.rpm_manifest.*.srpm_name", "include": True, "recursive": True} + ] + } + return self.find_container_images(image_request) diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index fbb0bc1..cdd7d68 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -387,6 +387,70 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): cont_repos.assert_called_with(expected_repo_request) self.assertEqual(ret, cont_repos.return_value) + @patch('freshmaker.lightblue.LightBlue.find_container_images') + @patch('os.path.exists') + def test_images_with_included_srpm(self, exists, + cont_images): + + exists.return_value = True + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + repositories = self.fake_repositories_with_content_sets + cont_images.return_value = self.fake_images_with_parsed_data + ret = lb.find_images_with_included_srpm(repositories, + "openssl") + + expected_image_request = { + "objectType": "containerImage", + "query": { + "$and": [ + { + "$or": [ + { + "field": "repositories.*.repository", + "op": "=", + "rvalue": "product/repo1" + }, + { + "field": "repositories.*.repository", + "op": "=", + "rvalue": "product2/repo2" + }, + ], + }, + { + "field": "repositories.*.published", + "op": "=", + "rvalue": True + }, + { + "field": "repositories.*.tags.*.name", + "op": "=", + "rvalue": "latest" + }, + { + "field": "parsed_data.rpm_manifest.*.srpm_name", + "op": "=", + "rvalue": "openssl" + }, + { + "field": "parsed_data.files.*.key", + "op": "=", + "rvalue": "buildfile" + } + ] + }, + "projection": [ + {"field": "brew", "include": True, "recursive": True}, + {"field": "parsed_data.files", "include": True, "recursive": True}, + {"field": "parsed_data.rpm_manifest.*.srpm_nevra", "include": True, "recursive": True}, + {"field": "parsed_data.rpm_manifest.*.srpm_name", "include": True, "recursive": True} + ] + } + cont_images.assert_called_with(expected_image_request) + self.assertEqual(ret, cont_images.return_value) + class TestEntityVersion(unittest.TestCase): """Test case for ensuring correct entity version in request""" From 499652f36697ae1c6d0c923801017298a05630b8 Mon Sep 17 00:00:00 2001 From: Stanislav Ochotnicky Date: Jun 01 2017 08:21:13 +0000 Subject: [PATCH 3/3] Add method for content set and package image search We combine earlier methods to find images which include content from source rpm from given content set. --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index d667e19..c2c4731 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -312,3 +312,53 @@ class LightBlue(object): ] } return self.find_container_images(image_request) + + + def find_images_with_package_from_content_set(self, srpm_name, + content_sets, + published=True, + deprecated=False, + release_category="Generally Available"): + """Query lightblue and find containers which contain given + package from one of content sets + + :param str srpm_name: srpm_name (source rpm name) to look for + :param list content_sets: list of strings (content sets) to consider + when looking for the packages + + :return: a list of dictionaries with three keys - repository, commit and + srpm_nevra. Repository is a name git repository including the + namespace. Commit is a git ref - usually a git commit + hash. srpm_nevra is whole NEVRA of source rpm that is included in + the given image - can be used for comparisons if needed + :rtype: list + """ + repos = self.find_repositories_with_content_sets(content_sets, + published=published, + deprecated=deprecated, + release_category=release_category) + if not repos: + return [] + images = self.find_images_with_included_srpm(repos, + srpm_name, + published=published) + commits = [] + for image in images: + for f in image["parsed_data"]["files"]: + if f['key'] == 'buildfile': + dockerfile_url = f['content_url'] + break + + for rpm in image["parsed_data"]["rpm_manifest"]: + print(rpm) + if rpm["srpm_name"] == srpm_name: + srpm_nevra = rpm['srpm_nevra'] + break + + dockerfile, _, commit = dockerfile_url.partition("?id=") + _, _, reponame = dockerfile.partition("/cgit/") + reponame = reponame.replace("/plain/Dockerfile","") + commits.append({"repository": reponame, + "commit": commit, + "srpm_nevra": srpm_nevra}) + return commits diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index cdd7d68..ca16dba 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -23,8 +23,7 @@ import json import unittest -from mock import call -from mock import patch +from mock import call, patch, Mock from six.moves import http_client from freshmaker.lightblue import ContainerImage @@ -452,6 +451,69 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): self.assertEqual(ret, cont_images.return_value) + @patch('freshmaker.lightblue.LightBlue.find_container_repositories') + @patch('freshmaker.lightblue.LightBlue.find_container_images') + @patch('os.path.exists') + def test_images_with_content_set_packages(self, exists, + cont_images, + cont_repos): + + exists.return_value = True + cont_repos.return_value = self.fake_repositories_with_content_sets + cont_images.return_value = self.fake_images_with_parsed_data + + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + ret = lb.find_images_with_package_from_content_set("openssl", + ["dummy-content-set-1"]) + + self.assertEqual(2,len(ret)) + self.assertEqual(ret, + [ + { + "repository": "rpms/repo-1", + "commit": "commit_hash1", + "srpm_nevra": "openssl-0:1.2.3-1.src" + }, + { + "repository": "ns/repo-2", + "commit": "commit_hash2", + "srpm_nevra": "openssl-1:1.2.3-1.src" + } + ]) + + @patch('freshmaker.lightblue.LightBlue.find_container_repositories') + @patch('freshmaker.lightblue.LightBlue.find_container_images') + @patch('os.path.exists') + def test_images_with_content_set_packages_exception(self, exists, + cont_images, + cont_repos): + + exists.return_value = True + cont_repos.side_effect = LightBlueRequestFailure( + {"errors": [{"msg": "dummy error"}]}, http_client.REQUEST_TIMEOUT) + cont_images.return_value = self.fake_images_with_parsed_data + + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + with self.assertRaises(LightBlueRequestFailure): + lb.find_images_with_package_from_content_set( + "openssl", + ["dummy-content-set-1"]) + + cont_repos.return_value = self.fake_repositories_with_content_sets + cont_images.side_effect = LightBlueRequestFailure( + {"errors": [{"msg": "dummy error"}]}, http_client.REQUEST_TIMEOUT) + + with self.assertRaises(LightBlueRequestFailure): + lb.find_images_with_package_from_content_set( + "openssl", + ["dummy-content-set-1"]) + + + class TestEntityVersion(unittest.TestCase): """Test case for ensuring correct entity version in request"""