From 0c813beac6b47070e4f181c386309acbdaad1154 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jan 22 2018 10:28:02 +0000 Subject: [PATCH 1/3] Allow querying unpublished Lightblue images. --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 2910496..fe8f00c 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -306,6 +306,64 @@ class LightBlue(object): self.entity_versions = entity_versions or {} + self.published = True + self.deprecated = False + self.release_category = "Generally Available" + + @property + def published(self): + """ + Returns the current value of "published" property. + """ + return self._published + + @published.setter + def published(self, value): + """ + When set to True, only published images are returned by LB queries. + When set to False, only unpublished images are returned by LB queries. + When set to None, any image (published or unpublished) can be returned + by LB queries. + """ + self._published = value + + @property + def deprecated(self): + """ + Returns the current value of "deprecated" property. + """ + return self._deprecated + + @deprecated.setter + def deprecated(self, value): + """ + When set to True, only deprecated images are returned by LB queries. + When set to False, only non-deprecated images are returned by LB + queries. + When set to None, any image (deprecated or non-deprecated) can be + returned by LB queries. + """ + self._deprecated = value + + @property + def release_category(self): + """ + Returns the current value of "release_category" property. + """ + return self._release_category + + @release_category.setter + def release_category(self, value): + """ + When set to non-empty string, LB queries will return only + repositories with specific release category. Options: + - "Deprecated" + - "Generally Available" + - "Beta" + - "Tech Preview" + """ + self._release_category = value + def _get_entity_version(self, entity_name): """Lookup configured entity's version @@ -394,23 +452,42 @@ class LightBlue(object): images.append(image) return images - def find_repositories_with_content_sets(self, - content_sets, - published=True, - deprecated=False, - release_category="Generally Available"): + def _set_container_repository_filters(self, request): + """ + Sets the additional filters to containerRepository request + based on the self.published, self.deprecated and self.release_category + attributes. + """ + if self.published is not None: + request["query"]["$and"].append({ + "field": "published", + "op": "=", + "rvalue": self.published + }) + + if self.deprecated is not None: + request["query"]["$and"].append({ + "field": "deprecated", + "op": "=", + "rvalue": self.deprecated + }) + + if self.release_category: + request["query"]["$and"].append({ + "field": "release_categories.*", + "op": "=", + "rvalue": self.release_category + }) + + return request + + def find_repositories_with_content_sets(self, content_sets): """Query lightblue and find containerRepositories which have content from at least one of the content_sets. By default ignore unpublished, deprecated repos or non-GA repositories :param list content_sets: list of strings (content sets) to consider when looking for the packages - :param bool published: whether to limit queries to published - repositories - :param bool deprecated: set to True to limit results to deprecated - repositories - :param str release_category: filter only repositories with specific - release category (options: Deprecated, Generally Available, Beta, Tech Preview) """ repo_request = { "objectType": "containerRepository", @@ -423,21 +500,6 @@ class LightBlue(object): "rvalue": c } for c in content_sets] }, - { - "field": "published", - "op": "=", - "rvalue": published - }, - { - "field": "deprecated", - "op": "=", - "rvalue": deprecated - }, - { - "field": "release_categories.*", - "op": "=", - "rvalue": release_category - } ] }, "projection": [ @@ -445,25 +507,17 @@ class LightBlue(object): {"field": "content_sets", "include": True, "recursive": True} ] } + + repo_request = self._set_container_repository_filters(repo_request) return self.find_container_repositories(repo_request) - def find_content_sets_for_repository(self, - repository, - published=True, - deprecated=False, - release_category="Generally Available"): + def find_content_sets_for_repository(self, repository): """ Query lightblue and find content sets which are used for Container image in repository `repository` :param str repository: name of the repository for which the content sets will be returned - :param bool published: whether to limit queries to published - repositories - :param bool deprecated: set to True to limit results to deprecated - repositories - :param str release_category: filter only repositories with specific - release category (options: Deprecated, Generally Available, Beta, Tech Preview) :return: list of found content sets, each of which is content set name. Empty list is returned if no repository is found. :rtype: list @@ -477,27 +531,14 @@ class LightBlue(object): "op": "=", "rvalue": repository }, - { - "field": "published", - "op": "=", - "rvalue": published - }, - { - "field": "deprecated", - "op": "=", - "rvalue": deprecated - }, - { - "field": "release_categories.*", - "op": "=", - "rvalue": release_category - } ] }, "projection": [ {"field": "content_sets", "include": True, "recursive": True} ] } + + repo_request = self._set_container_repository_filters(repo_request) repos = self.find_container_repositories(repo_request) if not repos: return [] @@ -519,8 +560,21 @@ class LightBlue(object): {"field": "repositories.*.repository", "include": True, "recursive": True}, ] - def find_images_with_included_srpm(self, repositories, srpm_name, - published=True): + def _set_container_image_filters(self, request): + """ + Sets the additional filters to containerImage request + based on the self.published attribute. + """ + if self.published is not None: + request["query"]["$and"].append({ + "field": "repositories.*.published", + "op": "=", + "rvalue": self.published + }) + + return request + + def find_images_with_included_srpm(self, repositories, srpm_name): """Query lightblue and find containerImages in given containerRepositories. By default limit only to images which have been @@ -528,8 +582,6 @@ class LightBlue(object): :param dict repositories: dictionary with repository names to look inside :param str srpm_name: srpm_name (source rpm name) to look for - :param bool published: whether to limit queries to images with at least - one published repository """ image_request = { "objectType": "containerImage", @@ -543,11 +595,6 @@ class LightBlue(object): } for r in repositories] }, { - "field": "repositories.*.published", - "op": "=", - "rvalue": published - }, - { "field": "repositories.*.tags.*.name", "op": "=", "rvalue": "latest" @@ -566,6 +613,7 @@ class LightBlue(object): }, "projection": self._get_default_projection() } + image_request = self._set_container_image_filters(image_request) return self.find_container_images(image_request) def find_unpublished_image_for_build(self, build): @@ -741,8 +789,7 @@ class LightBlue(object): images.append(image) def find_images_with_package_from_content_set( - self, srpm_name, content_sets, published=True, deprecated=False, - release_category="Generally Available", filter_fnc=None): + self, srpm_name, content_sets, filter_fnc=None): """Query lightblue and find containers which contain given package from one of content sets @@ -763,15 +810,10 @@ class LightBlue(object): the given image - can be used for comparisons if needed :rtype: list """ - repos = self.find_repositories_with_content_sets(content_sets, - published=published, - deprecated=deprecated, - release_category=release_category) + repos = self.find_repositories_with_content_sets(content_sets) if not repos: return [] - images = self.find_images_with_included_srpm(repos, - srpm_name, - published=published) + images = self.find_images_with_included_srpm(repos, srpm_name) # Filter out images based on the filter_fnc. if filter_fnc: @@ -785,8 +827,7 @@ class LightBlue(object): return images def find_images_to_rebuild( - self, srpm_name, content_sets, published=True, deprecated=False, - release_category="Generally Available", filter_fnc=None): + self, srpm_name, content_sets, filter_fnc=None): """ Find images to rebuild through image build layers @@ -807,8 +848,7 @@ class LightBlue(object): Freshmaker configuration. """ images = self.find_images_with_package_from_content_set( - srpm_name, content_sets, published, deprecated, release_category, - filter_fnc=filter_fnc) + srpm_name, content_sets, filter_fnc=filter_fnc) def _get_images_to_rebuild(image): """ diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index 2960275..0d45866 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -648,11 +648,6 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): ], }, { - "field": "repositories.*.published", - "op": "=", - "rvalue": True - }, - { "field": "repositories.*.tags.*.name", "op": "=", "rvalue": "latest" @@ -666,7 +661,12 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): "field": "parsed_data.files.*.key", "op": "=", "rvalue": "buildfile" - } + }, + { + "field": "repositories.*.published", + "op": "=", + "rvalue": True + }, ] }, "projection": lb._get_default_projection() From 0370d640fee35f13a157edade4eb0c596d683e87 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jan 22 2018 12:42:39 +0000 Subject: [PATCH 2/3] Return only the latest images when querying LB for unpublished container images. --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index fe8f00c..1b49276 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -32,6 +32,7 @@ from six.moves import http_client import concurrent.futures from freshmaker import log, conf from freshmaker.kojiservice import koji_service +import koji class LightBlueError(Exception): @@ -815,6 +816,25 @@ class LightBlue(object): return [] images = self.find_images_with_included_srpm(repos, srpm_name) + # In case we query for unpublished images, we need to return just + # the latest NVR for given name-version, otherwise images would + # contain all the versions which ever containing the srpm_name. + if not self.published: + # Sort images by brew build NVR descending + sorted_images = sorted( + images, key=lambda image: image['brew']['build'], reverse=True) + + # Iterate over all the images and only keep the very first one + # with the given name-version - this is the latest one. + images = [] + seen_name_versions = [] + for image in sorted_images: + parsed_build = koji.parse_NVR(image["brew"]["build"]) + nv = "%s-%s" % (parsed_build["name"], parsed_build["version"]) + if nv not in seen_name_versions: + images.append(image) + seen_name_versions.append(nv) + # Filter out images based on the filter_fnc. if filter_fnc: images = [image for image in images if not filter_fnc(image)] From 189ac859e85f2a356f5eaeabe60ebd1c47ccb396 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jan 22 2018 13:48:12 +0000 Subject: [PATCH 3/3] Use kwargs for published/deprecated/release_category --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 1b49276..37cca31 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -216,7 +216,9 @@ class ContainerImage(dict): data["srpm_nevra"] = srpm_nevra self.update(data) - def resolve_content_sets(self, lb_instance, children=None): + def resolve_content_sets( + self, lb_instance, children=None, published=True, + deprecated=False, release_category="Generally Available"): """ Find out the content_sets this image uses and store it as "content_sets" key in image. @@ -226,6 +228,12 @@ class ContainerImage(dict): :param list children: List of children to take the content_sets from in case this container image is unpublished and therefore without repositories from which we could get the list of content_sets. + :param bool published: whether to limit queries to published + repositories + :param bool deprecated: set to True to limit results to deprecated + repositories + :param str release_category: filter only repositories with specific + release category (options: Deprecated, Generally Available, Beta, Tech Preview) """ if "repositories" not in self or len(self["repositories"]) == 0: if not children: @@ -239,7 +247,8 @@ class ContainerImage(dict): # The child['content_sets'] should be always set for children # passed here, but in case it is not, just try it. if "content_sets" not in child: - child.resolve_content_sets(lb_instance) + child.resolve_content_sets(lb_instance, None, published, + deprecated, release_category) if not child["content_sets"]: continue @@ -260,7 +269,8 @@ class ContainerImage(dict): # is in multiple repositories, the content_sets of all of them # must be the same by definition. image_content_sets = lb_instance.find_content_sets_for_repository( - self["repositories"][0]["repository"]) + self["repositories"][0]["repository"], published, deprecated, + release_category) log.info("Container image %s uses following content sets: %r", self["brew"]["build"], image_content_sets) @@ -307,64 +317,6 @@ class LightBlue(object): self.entity_versions = entity_versions or {} - self.published = True - self.deprecated = False - self.release_category = "Generally Available" - - @property - def published(self): - """ - Returns the current value of "published" property. - """ - return self._published - - @published.setter - def published(self, value): - """ - When set to True, only published images are returned by LB queries. - When set to False, only unpublished images are returned by LB queries. - When set to None, any image (published or unpublished) can be returned - by LB queries. - """ - self._published = value - - @property - def deprecated(self): - """ - Returns the current value of "deprecated" property. - """ - return self._deprecated - - @deprecated.setter - def deprecated(self, value): - """ - When set to True, only deprecated images are returned by LB queries. - When set to False, only non-deprecated images are returned by LB - queries. - When set to None, any image (deprecated or non-deprecated) can be - returned by LB queries. - """ - self._deprecated = value - - @property - def release_category(self): - """ - Returns the current value of "release_category" property. - """ - return self._release_category - - @release_category.setter - def release_category(self, value): - """ - When set to non-empty string, LB queries will return only - repositories with specific release category. Options: - - "Deprecated" - - "Generally Available" - - "Beta" - - "Tech Preview" - """ - self._release_category = value - def _get_entity_version(self, entity_name): """Lookup configured entity's version @@ -453,42 +405,58 @@ class LightBlue(object): images.append(image) return images - def _set_container_repository_filters(self, request): + def _set_container_repository_filters( + self, request, published=True, deprecated=False, + release_category="Generally Available"): """ Sets the additional filters to containerRepository request based on the self.published, self.deprecated and self.release_category attributes. - """ - if self.published is not None: + :param bool published: whether to limit queries to published + repositories + :param bool deprecated: set to True to limit results to deprecated + repositories + :param str release_category: filter only repositories with specific + release category (options: Deprecated, Generally Available, Beta, Tech Preview) + """ + if published is not None: request["query"]["$and"].append({ "field": "published", "op": "=", - "rvalue": self.published + "rvalue": published }) - if self.deprecated is not None: + if deprecated is not None: request["query"]["$and"].append({ "field": "deprecated", "op": "=", - "rvalue": self.deprecated + "rvalue": deprecated }) - if self.release_category: + if release_category: request["query"]["$and"].append({ "field": "release_categories.*", "op": "=", - "rvalue": self.release_category + "rvalue": release_category }) return request - def find_repositories_with_content_sets(self, content_sets): + def find_repositories_with_content_sets( + self, content_sets, published=True, deprecated=False, + release_category="Generally Available"): """Query lightblue and find containerRepositories which have content from at least one of the content_sets. By default ignore unpublished, deprecated repos or non-GA repositories :param list content_sets: list of strings (content sets) to consider when looking for the packages + :param bool published: whether to limit queries to published + repositories + :param bool deprecated: set to True to limit results to deprecated + repositories + :param str release_category: filter only repositories with specific + release category (options: Deprecated, Generally Available, Beta, Tech Preview) """ repo_request = { "objectType": "containerRepository", @@ -509,16 +477,21 @@ class LightBlue(object): ] } - repo_request = self._set_container_repository_filters(repo_request) + repo_request = self._set_container_repository_filters( + repo_request, published, deprecated, release_category) return self.find_container_repositories(repo_request) - def find_content_sets_for_repository(self, repository): + def find_content_sets_for_repository( + self, repository, published=True, deprecated=False, + release_category="Generally Available"): """ Query lightblue and find content sets which are used for Container image in repository `repository` :param str repository: name of the repository for which the content sets will be returned + :param bool published: whether to limit queries to published + repositories :return: list of found content sets, each of which is content set name. Empty list is returned if no repository is found. :rtype: list @@ -539,7 +512,8 @@ class LightBlue(object): ] } - repo_request = self._set_container_repository_filters(repo_request) + repo_request = self._set_container_repository_filters( + repo_request, published, deprecated, release_category) repos = self.find_container_repositories(repo_request) if not repos: return [] @@ -561,26 +535,31 @@ class LightBlue(object): {"field": "repositories.*.repository", "include": True, "recursive": True}, ] - def _set_container_image_filters(self, request): + def _set_container_image_filters(self, request, published): """ Sets the additional filters to containerImage request based on the self.published attribute. + :param bool published: whether to limit queries to published + repositories """ - if self.published is not None: + if published is not None: request["query"]["$and"].append({ "field": "repositories.*.published", "op": "=", - "rvalue": self.published + "rvalue": published }) return request - def find_images_with_included_srpm(self, repositories, srpm_name): + def find_images_with_included_srpm(self, repositories, srpm_name, + published=True): """Query lightblue and find containerImages in given containerRepositories. By default limit only to images which have been published to at least one repository and images which have latest tag. + :param bool published: whether to limit queries to published + repositories :param dict repositories: dictionary with repository names to look inside :param str srpm_name: srpm_name (source rpm name) to look for """ @@ -614,7 +593,8 @@ class LightBlue(object): }, "projection": self._get_default_projection() } - image_request = self._set_container_image_filters(image_request) + image_request = self._set_container_image_filters( + image_request, published) return self.find_container_images(image_request) def find_unpublished_image_for_build(self, build): @@ -710,7 +690,9 @@ class LightBlue(object): return images[0] - def find_parent_images_with_package(self, child_image, srpm_name, layers): + def find_parent_images_with_package( + self, child_image, srpm_name, layers, published=True, + deprecated=False, release_category="Generally Available"): """ Returns the chain of all parent images of the image with parsed_data.layers `layers` which contain the package `srpm_name` @@ -756,7 +738,8 @@ class LightBlue(object): srpm_name=srpm_name) children = images if images else [child_image] if image: - image.resolve_content_sets(self, children=children) + image.resolve_content_sets(self, children, published, + deprecated, release_category) image.resolve_commit(srpm_name) if images: @@ -782,7 +765,9 @@ class LightBlue(object): images[-1]['error'] = err if parent: - parent.resolve_content_sets(self, children=images) + parent.resolve_content_sets( + self, images, published, deprecated, + release_category) parent.resolve_commit(srpm_name) images[-1]['parent'] = parent if not image: @@ -790,7 +775,9 @@ class LightBlue(object): images.append(image) def find_images_with_package_from_content_set( - self, srpm_name, content_sets, filter_fnc=None): + self, srpm_name, content_sets, filter_fnc=None, + published=True, deprecated=False, + release_category="Generally Available"): """Query lightblue and find containers which contain given package from one of content sets @@ -803,6 +790,12 @@ class LightBlue(object): will not be considered for a rebuild as well as its parent images. This function is used to filter out images not allowed by Freshmaker configuration. + :param bool published: whether to limit queries to published + repositories + :param bool deprecated: set to True to limit results to deprecated + repositories + :param str release_category: filter only repositories with specific + release category (options: Deprecated, Generally Available, Beta, Tech Preview) :return: a list of dictionaries with three keys - repository, commit and srpm_nevra. Repository is a name git repository including the @@ -811,15 +804,16 @@ class LightBlue(object): the given image - can be used for comparisons if needed :rtype: list """ - repos = self.find_repositories_with_content_sets(content_sets) + repos = self.find_repositories_with_content_sets( + content_sets, published, deprecated, release_category) if not repos: return [] - images = self.find_images_with_included_srpm(repos, srpm_name) + images = self.find_images_with_included_srpm(repos, srpm_name, published) # In case we query for unpublished images, we need to return just # the latest NVR for given name-version, otherwise images would # contain all the versions which ever containing the srpm_name. - if not self.published: + if not published: # Sort images by brew build NVR descending sorted_images = sorted( images, key=lambda image: image['brew']['build'], reverse=True) @@ -842,12 +836,14 @@ class LightBlue(object): for image in images: # We do not set "children" here in resolve_content_sets call, because # published images should have the content_set set. - image.resolve_content_sets(self) + image.resolve_content_sets(self, None, published, deprecated, + release_category) image.resolve_commit(srpm_name) return images def find_images_to_rebuild( - self, srpm_name, content_sets, filter_fnc=None): + self, srpm_name, content_sets, published=True, deprecated=False, + release_category="Generally Available", filter_fnc=None): """ Find images to rebuild through image build layers @@ -860,6 +856,12 @@ class LightBlue(object): :param str srpm_name: srpm_name (source rpm name) to look for :param list content_sets: list of strings (content sets) to consider when looking for the packages + :param bool published: whether to limit queries to published + repositories + :param bool deprecated: set to True to limit results to deprecated + repositories + :param str release_category: filter only repositories with specific + release category (options: Deprecated, Generally Available, Beta, Tech Preview) :param function filter_fnc: Function called as filter_fnc(container_image) with container_image being ContainerImage instance. If this function returns True, the image @@ -868,7 +870,8 @@ class LightBlue(object): Freshmaker configuration. """ images = self.find_images_with_package_from_content_set( - srpm_name, content_sets, filter_fnc=filter_fnc) + srpm_name, content_sets, filter_fnc, published, deprecated, + release_category) def _get_images_to_rebuild(image): """ @@ -887,7 +890,9 @@ class LightBlue(object): else: parent = self.get_image_by_layer(layers[1], len(layers) - 1) if parent: - parent.resolve_content_sets(self, children=[image]) + parent.resolve_content_sets( + self, [image], published, deprecated, + release_category) parent.resolve_commit(srpm_name) elif len(layers) != 2: err = "Cannot find parent of image %s with layer %s " \