From d61c397165819e6b11ef82f05a57a78d832135c4 Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: May 06 2020 15:39:58 +0000 Subject: [PATCH 1/6] PEP8 compatible formatting Signed-off-by: Tomas Hrcka --- diff --git a/fedora-minimal-compose.py b/fedora-minimal-compose.py index d447701..852c0a3 100755 --- a/fedora-minimal-compose.py +++ b/fedora-minimal-compose.py @@ -1,11 +1,11 @@ #!/usr/bin/python3 -#import fedora_messaging.api -#import json +# import fedora_messaging.api +# import json -#while True: +# while True: # m = fedora_messaging.api.Message(topic = 'org.fedoraproject.prod.buildsys.rpm.sign') - + # print(m.topic) # print(m.body) @@ -13,6 +13,7 @@ from fedora_messaging import api, config print(config.conf) + class SaveMessage(object): """ A fedora-messaging consumer that saves the message to a file. @@ -34,4 +35,5 @@ class SaveMessage(object): """ print(str(message)) + api.consume(SaveMessage) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/requirements.txt From c9302d743cc547738a5b4f14cc966fb67d568234 Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: May 06 2020 15:39:58 +0000 Subject: [PATCH 2/6] Add .gitignore --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..07b3b6a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea/ +venv/ + From de55785915a47efdccb0555c30ace009835a2144 Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: May 06 2020 15:39:58 +0000 Subject: [PATCH 3/6] Add requirements.txt and Dockerfile --- diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0dc613f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +FROM registry.fedoraproject.org/fedora:30 + +# set PYTHONUNBUFFERED env var to non-empty string so that our +# periods with no newline get printed immediately to the screen + +ENV PYTHONUNBUFFERED=true +# Get any latest updates since last container spin +RUN dnf update -y && dnf clean all +# Install pagure/fedmsg libraries + +RUN dnf -y install python3-ogr fedora-messaging && dnf clean all + +RUN mkdir /work + +WORKDIR /work + +# Copy the fedora config for fedora-messaging and also generate a random UUID +# https://fedora-messaging.readthedocs.io/en/latest/fedora-broker.html#getting-connected +# Note this will mean that if there is more than one container running +# using this image they will be reading from the same queue. Generally +# I expect this to only be running in one place. +RUN sed -e "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" /etc/fedora-messaging/fedora.toml > /work/my_config.toml + +# Set the Application Name +RUN sed -i 's|Example Application|Fedora Minimal Compose: https://pagure.io/releng/fmc|' /work/my_config.toml + +# Lower log levels to WARNING level +# Set the log config of fmc +RUN sed -i 's/\[log_config\]/\[log_config\]\nlevel = "DEBUG"/' /work/my_config.toml + +# Set the format for the log messages +RUN sed -i 's/format =.*$/format = "%(asctime)s %(levelname)s %(name)s - %(message)s"/' /work/my_config.toml + +# We only care about org.fedoraproject.prod.buildsys.rpm.sign messages +RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.buildsys.rpm.sign"]/' /work/my_config.toml +ADD fedora-minimal-compose.py /usr/lib/python3.7/site-packages/ + + + +# Call fedora-messaging CLI and tell it to use the ComposeTracker + +# class from the compose-tracker module. + +CMD fedora-messaging --conf /work/my_config.toml consume --callback=fedora-minimal-compose:SaveMessage \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29..1c8f48d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests==2.21.0 +fedora_messaging>=1.7.1 \ No newline at end of file From 81a99267917fa2c51710dd1a1286d7e61aa67dbe Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: May 06 2020 15:39:58 +0000 Subject: [PATCH 4/6] Add Readme Signed-off-by: Tomas Hrcka --- diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..edf1693 --- /dev/null +++ b/Readme.md @@ -0,0 +1,10 @@ +# Fedora Minimal Compose + +Service that checks fedora messiging and create new minimal composes accordingly + +# Run + +``` +[localhost]$ podman build . -t=fmc +[localhost]$ podman run localhost/fmc +``` \ No newline at end of file From ac2313dfc66c7b4e979d81fbba6b9d7d18866546 Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: May 06 2020 15:39:58 +0000 Subject: [PATCH 5/6] Add consumer object skeleton Signed-off-by: Tomas Hrcka --- diff --git a/Dockerfile b/Dockerfile index 0dc613f..7ece7a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,4 +41,4 @@ ADD fedora-minimal-compose.py /usr/lib/python3.7/site-packages/ # class from the compose-tracker module. -CMD fedora-messaging --conf /work/my_config.toml consume --callback=fedora-minimal-compose:SaveMessage \ No newline at end of file +CMD fedora-messaging --conf /work/my_config.toml consume --callback=fedora-minimal-compose:Consumer \ No newline at end of file diff --git a/Readme.md b/Readme.md index edf1693..9b5f5a9 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ # Fedora Minimal Compose -Service that checks fedora messiging and create new minimal composes accordingly +Service that checks fedora messaging and create new minimal composes accordingly # Run diff --git a/fedora-minimal-compose.py b/fedora-minimal-compose.py index 852c0a3..01b212b 100755 --- a/fedora-minimal-compose.py +++ b/fedora-minimal-compose.py @@ -1,39 +1,50 @@ #!/usr/bin/python3 -# import fedora_messaging.api -# import json - -# while True: -# m = fedora_messaging.api.Message(topic = 'org.fedoraproject.prod.buildsys.rpm.sign') - -# print(m.topic) -# print(m.body) - from fedora_messaging import api, config +import logging -print(config.conf) +# Set local logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +compose_threshold = 10 -class SaveMessage(object): - """ - A fedora-messaging consumer that saves the message to a file. +def number_of_packages_in_file(file): + return sum(1 for line in open(file)) - A single configuration key is used from fedora-messaging's - "consumer_config" key, "path", which is where the consumer will save - the messages:: - [consumer_config] - path = "/tmp/fedora-messaging/messages.txt" +class Consumer(object): + """ + A fedora-messaging consumer that process messages """ - def __call__(self, message): + def __init__(self): + """ + Configure the consumer """ - Invoked when a message is received by the consumer. - Args: - message (fedora_messaging.api.Message): The message from AMQP. + def get_nvr(self): + """ + Get nvr from message body for build + :return: """ - print(str(message)) + def process(self, message: api.Message): + """ + process the package build and decide if compose is required + :param message: + :return: + """ -api.consume(SaveMessage) + def __call__(self, message: api.Message): + """ + Invoked when a message is received by the consumer. + """ + logger.debug(message.topic) + logger.debug(message.body) + file = 'packages_to_compose' + if number_of_packages_in_file(file) >= compose_threshold: + # run compose + print(message.body['build']['nvr']) + else: + file.write(message.body['build']['nvr'] + "\n") diff --git a/packages_to_compose b/packages_to_compose new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/packages_to_compose From d64a744870b3152cd37d304e1a1c2e8e40c766fe Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: May 06 2020 15:39:58 +0000 Subject: [PATCH 6/6] More work --- diff --git a/Dockerfile b/Dockerfile index 7ece7a0..21ad927 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=true RUN dnf update -y && dnf clean all # Install pagure/fedmsg libraries -RUN dnf -y install python3-ogr fedora-messaging && dnf clean all +RUN dnf -y install python3-ogr fedora-messaging python3-odcs-client && dnf clean all RUN mkdir /work diff --git a/fedora-minimal-compose.py b/fedora-minimal-compose.py index 01b212b..7d41659 100755 --- a/fedora-minimal-compose.py +++ b/fedora-minimal-compose.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 from fedora_messaging import api, config +from odcs.client.odcs import ODCS, AuthMech import logging # Set local logging @@ -8,11 +9,33 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) compose_threshold = 10 - def number_of_packages_in_file(file): - return sum(1 for line in open(file)) + return sum(1 for line in file) + +def save_nvr_to_file(nvr): + f = open('packages_to_compose', 'w+') + if number_of_packages_in_file(f) >= compose_threshold: + # run compose + logger.info('Running the compose') + else: + if f.readline().rstrip('\n') is nvr: + logger.info('Package already in the set for compose') + else: + logger.info('Package writen to set for compose:') + logger.info(nvr) + f.write(nvr + '\n') + + return True +def run_compose(): +#odcs = ODCS("https://odcs.stg.fedoraproject.org", +# auth_mech=AuthMech.OpenIDC, +# openidc_token="e0d6b2c3-4293-4468-8b8f-57ff42fa84c9_vUVbx6m6Rmyc8j1Ecw2QPFXi6RhM7QrT") +#source_type = "tag" +#sources = "rawhide" +#compose = odcs.new_compose(sources, source_type) + return True class Consumer(object): """ A fedora-messaging consumer that process messages @@ -23,28 +46,12 @@ class Consumer(object): Configure the consumer """ - def get_nvr(self): - """ - Get nvr from message body for build - :return: - """ - - def process(self, message: api.Message): - """ - process the package build and decide if compose is required - :param message: - :return: - """ - def __call__(self, message: api.Message): """ Invoked when a message is received by the consumer. """ logger.debug(message.topic) logger.debug(message.body) - file = 'packages_to_compose' - if number_of_packages_in_file(file) >= compose_threshold: - # run compose - print(message.body['build']['nvr']) - else: - file.write(message.body['build']['nvr'] + "\n") + package = message.body['build']['nvr'] + save_nvr_to_file(package) + run_compose() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1c8f48d..7cc2330 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests==2.21.0 -fedora_messaging>=1.7.1 \ No newline at end of file +fedora_messaging>=1.7.1 +odcs \ No newline at end of file diff --git a/test_fmc.py b/test_fmc.py new file mode 100644 index 0000000..5631605 --- /dev/null +++ b/test_fmc.py @@ -0,0 +1,16 @@ +import copy +import json +import logging + +import pytest +import fedora_messaging.api +import fedora_messaging.config + +def test_message_process(mocker, caplog): + assert True + +def test_nvr_save(mocker, caplog): + assert True + +def test_compose_treshold(mocker, caplog): + assert True \ No newline at end of file