From ad0f320d8481659c24a2b22bfbba12c1eec6d3df Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 1/11] Add more debugging


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py
index 413b94a..9e6cfee 100644
--- a/monitor_gating/runner.py
+++ b/monitor_gating/runner.py
@@ -45,8 +45,12 @@ def notify(topic, message):
         print(f"Fedora Messaging broker rejected message {msg.id}: {err}")
     except fedora_messaging.exceptions.ConnectionException as err:
         print(f"Error sending message {msg.id}: {err}")
+        print(f"topic: {topic}")
+        print(f"message: {message}")
     except Exception as err:
         print(f"Error sending fedora-messaging message: {err}")
+        print(f"topic: {topic}")
+        print(f"message: {message}")
 
 
 def _clean_up_side_tags(utils):
@@ -145,6 +149,8 @@ def schedule(conf):
     except Exception as err:
         print(f"Tests failed with: {err}", flush=True)
         print(sys.exc_info()[0])
+        print("-"*20)
+        print(sys.exc_info())
 
         notify(
             topic=f"multi-build.end.error", message={"runid": runid, "exception": err},

From c0848e574e405377599f2f20f6309a31d4214f11 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 2/11] Check that the request return a 2xx code before proceeding


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py
index 6b0c027..0018f09 100644
--- a/monitor_gating/utils.py
+++ b/monitor_gating/utils.py
@@ -60,12 +60,15 @@ def blocking_issues(project, tags):
     issues = []
     try:
         r = requests.get(api)
-        issues = r.json()["issues"]
-        if tags:
-            t = set(tags[1:])
-            issues = [i for i in issues if t & set(i["tags"])]
-        for i in issues:
-            print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}")
+        if not r.ok:
+            print(f"Failed to query: {api} -- returned : {r.status_code}")
+        else:
+            issues = r.json()["issues"]
+            if tags:
+                t = set(tags[1:])
+                issues = [i for i in issues if t & set(i["tags"])]
+            for i in issues:
+                print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}")
     except Exception as e:
         print(f"Error when querying pagure for blocking issues: {e}")
     return issues

From 7f4103058675eb9c3652e2c3286d5e00beb6b1d8 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 3/11] Add missing import to the runner.py


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py
index 9e6cfee..579b250 100644
--- a/monitor_gating/runner.py
+++ b/monitor_gating/runner.py
@@ -18,7 +18,7 @@ import toml
 
 from . import multi_builds
 from . import single_build
-from .utils import MonitoringUtils, blocking_issues, run_command
+from .utils import MonitoringUtils, blocking_issues, run_command, report_failure
 
 
 s = sched.scheduler(time.time, time.sleep)

From 3f41fd5afa58c143ab72de2a2a3fda29e76ed957 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 4/11] When notifying on errors, don't send the raw error object, send its text version


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py
index 579b250..e00a675 100644
--- a/monitor_gating/runner.py
+++ b/monitor_gating/runner.py
@@ -153,7 +153,7 @@ def schedule(conf):
         print(sys.exc_info())
 
         notify(
-            topic=f"multi-build.end.error", message={"runid": runid, "exception": err},
+            topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)},
         )
 
     delay_when_failing = conf["delay_when_failing"]

From d01a6881be52e341dbbe9cf5d42c16993bcf4f9c Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 5/11] Flush some prints


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py
index 0018f09..210282f 100644
--- a/monitor_gating/utils.py
+++ b/monitor_gating/utils.py
@@ -46,7 +46,7 @@ Full log:
 
     req = requests.post(url, data=data, headers=headers)
     if not req.ok:
-        print(f"Error when trying to open a ticket at: {url} to report the failure")
+        print(f"Error when trying to open a ticket at: {url} to report the failure", flush=True)
 
 
 def blocking_issues(project, tags):
@@ -61,16 +61,16 @@ def blocking_issues(project, tags):
     try:
         r = requests.get(api)
         if not r.ok:
-            print(f"Failed to query: {api} -- returned : {r.status_code}")
+            print(f"Failed to query: {api} -- returned : {r.status_code}", flush=True)
         else:
             issues = r.json()["issues"]
             if tags:
                 t = set(tags[1:])
                 issues = [i for i in issues if t & set(i["tags"])]
             for i in issues:
-                print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}")
+                print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}", flush=True)
     except Exception as e:
-        print(f"Error when querying pagure for blocking issues: {e}")
+        print(f"Error when querying pagure for blocking issues: {e}", flush=True)
     return issues
 
 

From e719e33d996f86cb37e3dfe25e62c436d968c432 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 6/11] Add some more debugging


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py
index e00a675..2ce25bd 100644
--- a/monitor_gating/runner.py
+++ b/monitor_gating/runner.py
@@ -149,8 +149,14 @@ def schedule(conf):
     except Exception as err:
         print(f"Tests failed with: {err}", flush=True)
         print(sys.exc_info()[0])
-        print("-"*20)
+        print("-"*60)
         print(sys.exc_info())
+        print("="*60)
+        import traceback
+        traceback.print_exc(file=sys.stdout)
+        print("-"*60)
+        traceback.print_stack()
+
 
         notify(
             topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)},

From f6380764467df3254ffa60853240a64944de9ee1 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 7/11] Fix appending an element to a list rather than trying to call the list


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py
index 210282f..68f6332 100644
--- a/monitor_gating/utils.py
+++ b/monitor_gating/utils.py
@@ -156,7 +156,7 @@ class MonitoringUtils:
             run_command([command, "switch-branch", f"{name}"], cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("fedpkg")
+            self.failed.append("fedpkg")
             self.print_user(info_log, success=False)
 
     def bump_release(self, name, folder):
@@ -168,7 +168,7 @@ class MonitoringUtils:
             run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("rpmdev-bumspec")
+            self.failed.append("rpmdev-bumspec")
             self.print_user(info_log, success=False)
 
     def commit_changes(self, commit_log, folder):
@@ -181,7 +181,7 @@ class MonitoringUtils:
             run_command(["git", "commit", "-asm", commit_log], cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("git")
+            self.failed.append("git")
             self.print_user(info_log, success=False)
 
     def push_changes(self, folder, target, branch, force=False):
@@ -196,7 +196,7 @@ class MonitoringUtils:
             run_command(cmd, cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("git/dist-git")
+            self.failed.append("git/dist-git")
             self.print_user(info_log, success=False)
 
     def pull_changes(self, folder, target, branch):
@@ -209,7 +209,7 @@ class MonitoringUtils:
             run_command(cmd, cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("git/dist-git")
+            self.failed.append("git/dist-git")
             self.print_user(info_log, success=False)
 
     def open_pullrequest(self, base_url, username, namespace, name, branch, token):
@@ -237,7 +237,7 @@ class MonitoringUtils:
             success = False
             pr_id = None
             pr_uid = None
-            self.failed("dist-git")
+            self.failed.append("dist-git")
         else:
             output = req.json()
             pr_id = str(output["id"])
@@ -260,7 +260,7 @@ class MonitoringUtils:
             self.print_user(info_log, success=True)
             return nevr.strip().decode("utf-8")
         except MonitoringException:
-            self.failed("fedpkg")
+            self.failed.append("fedpkg")
             self.print_user(info_log, success=False)
 
     def build_package(self, command, folder, target=None):
@@ -275,7 +275,7 @@ class MonitoringUtils:
             run_command(command, cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("koji")
+            self.failed.append("koji")
             self.print_user(info_log, success=False)
 
     def chain_build_packages(self, command, packages, folder, target=None):
@@ -295,7 +295,7 @@ class MonitoringUtils:
             run_command(command, cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("koji")
+            self.failed.append("koji")
             self.print_user(info_log, success=False)
 
     def get_build_tags(self, koji_url, nevr, expected_ends):
@@ -353,7 +353,7 @@ class MonitoringUtils:
 
         info_log = f"Retrieving koji tags: {tags}"
         if not success:
-            self.failed("koji")
+            self.failed.append("koji")
         self.print_user(info_log, success=success)
 
     def create_update(
@@ -387,7 +387,7 @@ class MonitoringUtils:
             run_command(command)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("bodhi")
+            self.failed.append("bodhi")
             self.print_user(info_log, success=False)
 
     def get_update_id(self, nevr, url):
@@ -408,7 +408,7 @@ class MonitoringUtils:
 
             if (datetime.datetime.utcnow() - start).seconds > (15 * 60):
                 success = False
-                self.failed("bodhi")
+                self.failed.append("bodhi")
                 info_log = f"Update for {nevr} not created within 15 minutes"
                 break
 
@@ -568,7 +568,7 @@ class MonitoringUtils:
         info_log += f" - ran for: {(end - start_lookup).seconds}s"
         self.print_user(info_log, success=success)
         if not success:
-            self.failed("datagrepper")
+            self.failed.append("datagrepper")
 
     def lookup_ci_resultsdb(self, nevr, name, url):
         """ Check the CI results in the specified resultsdb for results about
@@ -619,7 +619,7 @@ class MonitoringUtils:
         info_log += f" - ran for: {(end - start).seconds}s"
         self.print_user(info_log, success=success)
         if not success:
-            self.failed("resultsdb")
+            self.failed.append("resultsdb")
 
     def waive_update(self, command, updateid, prod=True, username=None, password=None):
         """ Waive all the tests results for the specified update using bodhi's
@@ -645,7 +645,7 @@ class MonitoringUtils:
             run_command(command)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed("waiverdb")
+            self.failed.append("waiverdb")
             self.print_user(info_log, success=False)
 
     def get_pr_flag(
@@ -701,7 +701,7 @@ class MonitoringUtils:
 
         self.print_user(info_log, success=success)
         if not success:
-            self.failed("dist-git")
+            self.failed.append("dist-git")
 
     def merge_pr(self, base_url, username, namespace, name, pr_id, token):
         """ Merge the specified PR
@@ -722,7 +722,7 @@ class MonitoringUtils:
 
         self.print_user(info_log, success=success)
         if not success:
-            self.failed("dist-git")
+            self.failed.append("dist-git")
 
     def finalize(self, start):
         """ End data returned. """
@@ -745,7 +745,7 @@ class MonitoringUtils:
             self.print_user(info_log, success=True)
         except (MonitoringException, Exception) as err:
             print(err)
-            self.failed("dist-git")
+            self.failed.append("dist-git")
             self.print_user(info_log, success=False)
         return side_tag_name
 

From d0bd9a0b5f6667fca6ad0e8e2f00318b2d2ef368 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 8/11] Fix typo in the rpmdev-bumpspec name


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py
index 68f6332..5a04128 100644
--- a/monitor_gating/utils.py
+++ b/monitor_gating/utils.py
@@ -168,7 +168,7 @@ class MonitoringUtils:
             run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder)
             self.print_user(info_log, success=True)
         except MonitoringException:
-            self.failed.append("rpmdev-bumspec")
+            self.failed.append("rpmdev-bumpspec")
             self.print_user(info_log, success=False)
 
     def commit_changes(self, commit_log, folder):

From e320728649f1591b60da434dc2819f9cf48edc85 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 9/11] Be more specific about the error


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py
index 5a04128..753ba69 100644
--- a/monitor_gating/utils.py
+++ b/monitor_gating/utils.py
@@ -47,6 +47,8 @@ Full log:
     req = requests.post(url, data=data, headers=headers)
     if not req.ok:
         print(f"Error when trying to open a ticket at: {url} to report the failure", flush=True)
+        print(f"Request to {url} with data={data} returned: {req.status_code}", flush=True)
+        print(f"Output: {req.text}", flush=True)
 
 
 def blocking_issues(project, tags):

From 9e59d09e7bc1cf787d7511402bd402dff5c458d9 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:54 +0000
Subject: [PATCH 10/11] Fix field name when opening an issue on pagure


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py
index 753ba69..3dddb04 100644
--- a/monitor_gating/utils.py
+++ b/monitor_gating/utils.py
@@ -37,7 +37,7 @@ Full log:
 
     data = {
         "title": title,
-        "content": content,
+        "issue_content": content,
         "tag": tag,
     }
     headers = {

From 6391facdd7ff99ce054fad9be31339c70c8f6dde Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Jul 07 2020 13:28:55 +0000
Subject: [PATCH 11/11] Fix returning the MonitorUtils object


Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/monitor_gating/multi_builds.py b/monitor_gating/multi_builds.py
index 9cbe830..e740330 100644
--- a/monitor_gating/multi_builds.py
+++ b/monitor_gating/multi_builds.py
@@ -123,7 +123,7 @@ def main(args, utils=None):
 
         if not updateid:
             utils.finalize(start)
-            return
+            return utils
 
         # Check that bodhi notified the pipeline it can run
         utils.lookup_results_datagrepper(

