From 22ce74a26a651bd2fd632c92c4bb95c735e9fd19 Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Feb 04 2020 10:13:37 +0000 Subject: [PATCH 1/3] Fixed multi-build, so that it conforms to the refactor, added more log messages to single build. --- diff --git a/monitor_gating.py b/monitor_gating.py index 2c30613..f39ac84 100644 --- a/monitor_gating.py +++ b/monitor_gating.py @@ -41,10 +41,6 @@ from utils import( _log = logging.getLogger(__name__) -class MonitoringException(Exception): - """The base class for all exceptions raised by this script.""" - - def get_arguments(args): """ Parse and return the CLI arguments. """ @@ -214,7 +210,7 @@ def main(args): else: updateid = args.update - # Check the tag of the build + utils.print_user("Check the tag of the build") utils.get_build_tags( conf.get("koji_hub"), nevr, @@ -225,7 +221,7 @@ def main(args): utils.finalize(start) return - # Check that bodhi notified the pipeline it can run + utils.print_user(" Check that bodhi notified the pipeline it can run") utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="bodhi to CI", @@ -234,14 +230,14 @@ def main(args): bodhi_id=updateid, ) - # Check that the CI pipeline is running + utils.print_user(" Check that the CI pipeline is running") utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (running)", topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", nevr=nevr, ) - # Check at the CI pipeline has completed + print_user(" Check at the CI pipeline has completed") utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (complete)", @@ -250,19 +246,19 @@ def main(args): duration=30, ) - # Check the tag of the build + print_user(" Check the tag of the build") utils.get_build_tags( conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], ) - # Check that the CI results made it to resultsdb + utils.print_user(" Check that the CI results made it to resultsdb") utils.lookup_ci_resultsdb( nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] ) - # Check that resultsdb announced the new results + utils.print_user(" Check that resultsdb announced the new results") utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="resultsdb", @@ -270,7 +266,7 @@ def main(args): nevr=nevr, ) - # Check that greenwave reacted to resultsdb's new results + utils.print_user(" Check that greenwave reacted to resultsdb's new results") utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="greenwave", @@ -278,7 +274,7 @@ def main(args): nevr=nevr, ) - # Check the tag of the build -- build is blocked but should be signed + utils.print_user(" Check the tag of the build -- build is blocked but should be signed") utils.get_build_tags( conf.get("koji_hub"), nevr, diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 02e96f1..5ff0769 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -27,42 +27,12 @@ import logging import os import sys import tempfile -import time import toml -import requests - -from utils import( - MonitoringException, - print_user, - run_command, - clone_repo, - add_remote, - switch_branch, - bump_release, - commit_changes, - chain_build_packages, - push_changes, - pull_changes, - open_pullrequest, - get_nevr, - build_package, - get_build_tags, - get_update_id, - lookup_results_datagrepper, - lookup_ci_resultsdb, - waive_update, - get_pr_flag, - merge_pr, - finalize, - create_update, -) - -_log = logging.getLogger(__name__) +from utils import MonitoringUtils -class MonitoringException(Exception): - """The base class for all exceptions raised by this script.""" +_log = logging.getLogger(__name__) def get_arguments(args): @@ -98,47 +68,6 @@ def get_arguments(args): return parser.parse_args(args) -def create_side_tag(command, folder): - """ Create a side-tag to build packages in it. """ - info_log = f"Creating the side-tag" - print_user(info_log) - command = [command, "request-side-tag"] - side_tag_name = None - try: - output = run_command(command, cwd=folder) - lines = output.decode("utf-8").split("\n") - _, side_tag_name, _ = lines[0].split("'") - info_log = f"Created side-tag {side_tag_name}" - print_user(info_log, success=True) - except (MonitoringException, Exception) as err: - print(err) - print_user(info_log, success=False) - return side_tag_name - - -def clone_and_bump(folder, nevrs, conf, name, target=None, new_side_tag=False): - """Clone the repo, bump the release, commit and push.""" - namespace = conf["namespace"] - branch = conf["branch"] - - clone_repo(conf["fedpkg"], namespace, name, folder=folder) - gitfolder = os.path.join(folder, name) - switch_branch(conf["fedpkg"], branch, folder=gitfolder) - side_tag_name = None - # Create a side-tag - if new_side_tag: - side_tag_name = create_side_tag(conf["fedpkg"], folder=gitfolder) - target = side_tag_name - bump_release(name, folder=gitfolder) - commit_changes("Bump release", folder=gitfolder) - nevr = get_nevr(conf["fedpkg"], folder=gitfolder) - nevrs[name] = nevr - # Push to the main repo - push_changes(gitfolder, "origin", branch) - print(f" Upcoming build : {nevr}") - return (nevrs, target) - - def main(args): """ Main method used by this script. """ start = datetime.datetime.utcnow() @@ -146,21 +75,20 @@ def main(args): args = get_arguments(args) conf = toml.load(args.conf) - - fas_username = conf["fas_username"] + utils = MonitoringUtils() with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: print(f"Working in {folder}\n") nevrs = {} # Bump the release on both packages: - nevrs, side_tag_name = clone_and_bump( + nevrs, side_tag_name = utils.clone_and_bump( folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True) - nevrs, _ = clone_and_bump( + nevrs, _ = utils.clone_and_bump( folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name) # Chain-build the packages - chain_build_packages( + utils.chain_build_packages( conf["fedpkg"], packages=conf["name_multi_1"], folder=os.path.join(folder, conf["name_multi_2"]), @@ -168,10 +96,10 @@ def main(args): ) if args.conflict: - clone_to_build(folder, nevrs, conf, name, target=None) + utils.clone_to_build(folder, nevrs, conf, conf["name_multi_1"], target=None) # Create the update - create_update( + utils.create_update( conf["bodhi-cli"], side_tag_name, prod=conf["_env"] == "prod", @@ -179,25 +107,25 @@ def main(args): password=conf.get("bodhi-password"), from_tag=True, ) - updateid = get_update_id( + updateid = utils.get_update_id( nevrs[list(nevrs.keys())[0]], conf["bodhi"], ) print(f" Update created : {updateid}") # Check the tag of the build - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevrs[conf["name_multi_1"]], expected_ends=["signing-pending", "testing-pending"], ) if not updateid: - finalize(start) + utils.finalize(start) return # Check that bodhi notified the pipeline it can run - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="bodhi to CI", topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." @@ -213,7 +141,7 @@ def main(args): nevr_names.append(nevr) # Check that the CI pipeline is running - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (running)", topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", @@ -221,7 +149,7 @@ def main(args): start=start_dg, ) # Check at the CI pipeline has completed - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (complete)", topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", @@ -230,19 +158,19 @@ def main(args): ) # Check the tag of the build - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], ) # Check that the CI results made it to resultsdb - lookup_ci_resultsdb( + utils.lookup_ci_resultsdb( nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] ) # Check that resultsdb announced the new results - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="resultsdb", topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", @@ -251,7 +179,7 @@ def main(args): ) # Check that greenwave reacted to resultsdb's new results - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="greenwave", topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", @@ -260,7 +188,7 @@ def main(args): ) # Check the tag of the build -- build is blocked but should be signed - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], @@ -268,7 +196,7 @@ def main(args): if not args.no_waive: nevr = nevrs[list(nevrs.keys())[0]] - waive_update( + utils.waive_update( conf["bodhi-cli"], updateid, prod=conf["_env"] == "prod", @@ -277,7 +205,7 @@ def main(args): ) # Check that waiverdb announced the new waiver - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="waiverdb", topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", @@ -285,7 +213,7 @@ def main(args): ) # Check that greenwave reacted to the new waiver - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="greenwave", topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", @@ -293,13 +221,13 @@ def main(args): ) # Check the tag of the build -- build was waived, let is through - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevr, expected_ends=["f32"], ) - finalize(start) + utils.finalize(start) if __name__ == "__main__": diff --git a/utils.py b/utils.py index afd629c..014fbdd 100644 --- a/utils.py +++ b/utils.py @@ -128,7 +128,7 @@ class MonitoringUtils: def pull_changes(self, folder, target, branch): """ Pull all changes using git. """ - info_log = f"Pushing changes" + info_log = f"Pulling changes" self.print_user(info_log) try: cmd = ["git", "pull", "--rebase", target, branch] @@ -377,6 +377,13 @@ class MonitoringUtils: end_url = url end_url += f"&page={page}" data = requests.get(end_url).json() + if "raw_messages" not in data: + nomsg_log = f"No messages in data-grepper on {end_url} " + if 'error' in data: + nomsg_log += data['error'] + self.print_user(nomsg_log) + break + for message in data["raw_messages"]: # Old message format from the CI pipeline @@ -644,6 +651,46 @@ class MonitoringUtils: self.logs.append(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") print(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") + def create_side_tag(self, command, folder): + """ Create a side-tag to build packages in it. """ + info_log = f"Creating the side-tag" + self.print_user(info_log) + command = [command, "request-side-tag"] + side_tag_name = None + try: + output = run_command(command, cwd=folder) + lines = output.decode("utf-8").split("\n") + _, side_tag_name, _ = lines[0].split("'") + info_log = f"Created side-tag {side_tag_name}" + self.print_user(info_log, success=True) + except (MonitoringException, Exception) as err: + print(err) + self.print_user(info_log, success=False) + return side_tag_name + + + def clone_and_bump(self, folder, nevrs, conf, name, target=None, new_side_tag=False): + """Clone the repo, bump the release, commit and push.""" + namespace = conf["namespace"] + branch = conf["branch"] + + self.clone_repo(conf["fedpkg"], namespace, name, folder=folder) + gitfolder = os.path.join(folder, name) + self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) + side_tag_name = None + # Create a side-tag + if new_side_tag: + side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder) + target = side_tag_name + self.bump_release(name, folder=gitfolder) + self.commit_changes("Bump release", folder=gitfolder) + nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder) + nevrs[name] = nevr + # Push to the main repo + self.push_changes(gitfolder, "origin", branch) + print(f" Upcoming build : {nevr}") + return (nevrs, target) + def run_command(command, cwd=None): """ Run the specified command in a specific working directory if one From 58bd0d430c58860a40eb8fd9ef8ca4c5fb5ed57c Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Feb 04 2020 10:58:48 +0000 Subject: [PATCH 2/3] Fixed missing utils. for some print_user calls, removed unused imports --- diff --git a/monitor_gating.py b/monitor_gating.py index f39ac84..800f6d5 100644 --- a/monitor_gating.py +++ b/monitor_gating.py @@ -27,16 +27,10 @@ import logging import os import sys import tempfile -import time import toml -import requests -from utils import( - MonitoringException, - MonitoringUtils, - run_command, -) +from utils import MonitoringUtils _log = logging.getLogger(__name__) @@ -237,7 +231,7 @@ def main(args): topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", nevr=nevr, ) - print_user(" Check at the CI pipeline has completed") + utils.print_user(" Check at the CI pipeline has completed") utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (complete)", @@ -246,7 +240,7 @@ def main(args): duration=30, ) - print_user(" Check the tag of the build") + utils.print_user(" Check the tag of the build") utils.get_build_tags( conf.get("koji_hub"), nevr, diff --git a/utils.py b/utils.py index 014fbdd..13645e5 100644 --- a/utils.py +++ b/utils.py @@ -10,10 +10,8 @@ import datetime import logging import os import subprocess -import tempfile import time -import toml import requests _log = logging.getLogger(__name__) From 5cc6221dd16b07dd709bdc95358a757bacb1721b Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Feb 04 2020 13:26:38 +0000 Subject: [PATCH 3/3] Added openshift project definition --- diff --git a/README.rst b/README.rst index e13860b..fa05722 100644 --- a/README.rst +++ b/README.rst @@ -19,3 +19,13 @@ Here are the steps it does: - Checks (via datagrepper) that waiverdb has announced the new waiver - Checks (via datagrepper) that greenwave has announced a decision +To run in openshift, you need a ssh-key and a keytab for the principal, +alongside the config. +Currently the deployment is hardcoded to run against services in staging. + +``` +oc create secret generic monitor-gating-config --from-file monitor_gating_stg.cfg +oc create secret generic ssh --from-file id_rsa +oc create secret generic keytab --from-file user.keytab +oc create -f project.yaml +``` \ No newline at end of file diff --git a/project.yaml b/project.yaml new file mode 100644 index 0000000..603821a --- /dev/null +++ b/project.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +items: +- apiVersion: apps.openshift.io/v1 + kind: DeploymentConfig + metadata: + labels: + app: "monitor-gating-runner" + name: monitor-gating-runner + spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + deploymentconfig: monitor-gating-runner + strategy: + activeDeadlineSeconds: 21600 + recreateParams: + timeoutSeconds: 600 + resources: {} + rollingParams: + intervalSeconds: 1 + maxSurge: 25% + maxUnavailable: 25% + timeoutSeconds: 600 + updatePeriodSeconds: 1 + type: Rolling + template: + metadata: + labels: + app: monitor-gating-runner + deploymentconfig: monitor-gating-runner + spec: + containers: + - args: + - -c + - kinit asaleh@STG.FEDORAPROJECT.ORG -kt /keytab/user.keytab && ssh-keyscan + pkgs.stg.fedoraproject.org >> /.ssh/known_hosts && cd /opt && USER=asaleh + python3 runner.py runner.cfg + command: + - sh + image: image-registry.openshift-image-registry.svc:5000/monitor-gating-poc-00/monitor-gating-runner@sha256:e517842b9cec141fbb7c90980f719c4ad4c718fb789848c12ff84745b3805b6a + imagePullPolicy: Always + name: monitor-gating + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /opt/monitor_gating_stg.cfg + name: config-volume + readOnly: true + subPath: monitor_gating_stg.cfg + - mountPath: /keytab + name: keytab-volume + readOnly: true + - mountPath: /.ssh + name: ssh-dir + - mountPath: /.ssh/id_rsa + name: ssh-volume + subPath: id_rsa + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - secret: + defaultMode: 365 + secretName: monitor-gating-config + name: config-volume + - name: keytab-volume + secret: + defaultMode: 365 + secretName: keytab + - name: ssh-volume + secret: + defaultMode: 365 + secretName: ssh + - emptyDir: {} + name: ssh-dir + test: false + triggers: + - type: ConfigChange + - imageChangeParams: + automatic: true + containerNames: + - monitor-gating + from: + kind: ImageStreamTag + name: monitor-gating-runner:latest + type: ImageChange +- apiVersion: build.openshift.io/v1 + kind: BuildConfig + metadata: + labels: + build: monitor-gating-runner + name: monitor-gating-runner + spec: + failedBuildsHistoryLimit: 5 + nodeSelector: null + output: + to: + kind: ImageStreamTag + name: monitor-gating-runner:latest + postCommit: {} + resources: {} + runPolicy: Serial + source: + dockerfile: "FROM registry.fedoraproject.org/fedora:31 \nLABEL maintainer=\"Adam + Saleh \"\nRUN dnf install -y python3-pip bodhi-client + fedpkg-stage\nCOPY . /opt\nRUN ls /opt\nRUN pip-3 install fedora-messaging + requests toml\n" + git: + ref: master + uri: https://pagure.io/fedora-ci/monitor-gating.git + type: Git + strategy: + dockerStrategy: {} + type: Docker + successfulBuildsHistoryLimit: 5 + triggers: [] +- apiVersion: image.openshift.io/v1 + kind: ImageStream + metadata: + name: monitor-gating-runner + spec: + lookupPolicy: + local: false + tags: + - annotations: null + generation: 1 + importPolicy: {} + name: latest + referencePolicy: + type: Source +kind: List +metadata: + resourceVersion: "" + selfLink: ""