From b4b3f6de06e836fa0f879cc3d7c5568b2c2c4e8b Mon Sep 17 00:00:00 2001
From: Andrei Paplauski <apaplaus@redhat.com>
Date: Jun 09 2020 12:34:04 +0000
Subject: [PATCH 1/2] Periodically reduce size of images to rebuild


When list of lists with images to be rebuilt is constructed, we will
sometimes call deduplication function on it. Because we need to reduce
its size during construction, not after it.

For start I've chosen to do deduplication every 50 new tuples of images,
because deduplication doesn't reduce amount of tuples,
it only replaces images in this tuples with newer ones.

Signed-off-by: Andrei Paplauski <apaplaus@redhat.com>

---

diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py
index ed984c9..9deec29 100644
--- a/freshmaker/lightblue.py
+++ b/freshmaker/lightblue.py
@@ -1531,17 +1531,22 @@ class LightBlue(object):
         # For every image, find out all its parent images which contain the
         # srpm_name package and store these lists to to_rebuild.
         to_rebuild = []
+        optimization_base = 50
         with ThreadPoolExecutor(max_workers=conf.max_thread_workers) as executor:
             for result in executor.map(_get_images_to_rebuild, images):
                 to_rebuild.extend(result.values())
-
+                # Memory consumption of fully constructed to_rebuild list could
+                # be large. To prevent this we will periodically use
+                # deduplication on the list to reduce it size.
+                if len(to_rebuild) > optimization_base:
+                    self._deduplicate_images_to_rebuild(to_rebuild)
+                    optimization_base += 50
         # The to_rebuild list now contains all the images which need to be
         # rebuilt, but there are lot of duplicates there.
 
         # At first remove duplicated images which share the same name and
         # version, but different release.
         to_rebuild = self._deduplicate_images_to_rebuild(to_rebuild)
-
         # Get all the directly affected images so that any parents that are not marked as
         # directly affected can be set in _images_to_rebuild_to_batches
         directly_affected_nvrs = {

From 8d5afc8f6dd428a93c790fdd4462ff5d4ed63f56 Mon Sep 17 00:00:00 2001
From: Andrei Paplauski <apaplaus@redhat.com>
Date: Jun 09 2020 12:34:04 +0000
Subject: [PATCH 2/2] Query Lightblue only for useful fields of rpms


Now we will not request unnecessary fields of rpms from Lightblue.
And by that reduce memory consumption of returned container images.

RESOLVES CLOUDWF-1612

---

diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py
index 9deec29..9932346 100644
--- a/freshmaker/lightblue.py
+++ b/freshmaker/lightblue.py
@@ -708,11 +708,22 @@ class LightBlue(object):
                              "field": "srpm_name",
                              "op": "=",
                              "rvalue": srpm_name
-                         } for srpm_name in srpm_names]}}]
+                         } for srpm_name in srpm_names]},
+                     "project": [
+                         {"field": "srpm_nevra", "include": True},
+                         {"field": "nvra", "include": True},
+                         {"field": "srpm_name", "include": True},
+                     ]
+                     }
+                ]
             else:
                 projection += [
-                    {"field": "rpm_manifest.*.rpms", "include": True, "recursive": True},
-                    {"field": "rpm_manifest.*.rpms.*.srpm_name", "include": True, "recursive": True},
+                    {"field": "rpm_manifest.*.rpms.*.srpm_nevra",
+                     "include": True, "recursive": True},
+                    {"field": "rpm_manifest.*.rpms.*.nvra",
+                     "include": True, "recursive": True},
+                    {"field": "rpm_manifest.*.rpms.*.srpm_name",
+                     "include": True, "recursive": True},
                 ]
         return projection
 
diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py
index 6bba125..74acace 100644
--- a/tests/test_lightblue.py
+++ b/tests/test_lightblue.py
@@ -1979,7 +1979,8 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase):
                             {'field': 'content_sets', 'include': True, 'recursive': True},
                             {'field': 'parent_brew_build', 'include': True, 'recursive': False},
                             {'field': 'architecture', 'include': True, 'recursive': False},
-                            {'field': 'rpm_manifest.*.rpms', 'include': True, 'recursive': True},
+                            {'field': 'rpm_manifest.*.rpms.*.srpm_nevra', 'include': True, 'recursive': True},
+                            {'field': 'rpm_manifest.*.rpms.*.nvra', 'include': True, 'recursive': True},
                             {'field': 'rpm_manifest.*.rpms.*.srpm_name', 'include': True, 'recursive': True}],
              'objectType': 'containerImage'})