From ac21de29896d481ae377061d0f417d6bc897b8a6 Mon Sep 17 00:00:00 2001 From: Luiz Carvalho Date: Apr 08 2020 20:34:29 +0000 Subject: Build container image without an RPM Prior to this commit, the process of building a container image for Freshmaker created a temporary RPM which was then installed in the container image. Given that the RPM is immediately discarded, and that Freshmaker is never distributed as an RPM, this process is unnecessarily complex. This commit removes the RPM build process completely and simply builds the container image directly from source. Because Freshmaker is not installed from an RPM, its contents are found in a different location. Previously, the freshmaker-* console scripts were installed under /usr/bin, but now they're under /usr/local/bin. This is also true for the python package. It has moved from /usr/lib/python3.7/site-packages/freshmaker to /usr/local/lib/python3.7/site-packages/freshmaker This change in path is expected as /usr/local is meant for content not provided by the Linux distribution itself. Also, man pages for the freshmaker-* console scripts are no longer generated. There appears to be no value in generating those. Signed-off-by: Luiz Carvalho wip: will this work?! Signed-off-by: Luiz Carvalho --- diff --git a/.copr/Makefile b/.copr/Makefile deleted file mode 100644 index 70515e4..0000000 --- a/.copr/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -srpm: - /usr/bin/bash .copr/prepare_spec.sh - python setup.py sdist - cp ./dist/*.tar.gz ./.copr - touch ./.copr/sources - fedpkg --path ./.copr --release f28 srpm - cp ./.copr/*.src.rpm $(outdir) -clean: - rm -f .copr/*.tar.gz .copr/sources .copr/*.spec .copr/*.src.rpm diff --git a/.copr/freshmaker.spec.in b/.copr/freshmaker.spec.in deleted file mode 100644 index 8a709dc..0000000 --- a/.copr/freshmaker.spec.in +++ /dev/null @@ -1,132 +0,0 @@ -Name: freshmaker -Version: $FRESHMAKER_VERSION -Release: $FRESHMAKER_RELEASE%{?dist} -Summary: Freshmaker is a service scheduling rebuilds of artifacts as new content becomes available. - -Group: Development/Tools -License: MIT -URL: https://pagure.io/freshmaker -Source0: https://files.pythonhosted.org/packages/source/o/%{name}/%{name}-%{version}.tar.gz - -BuildArch: noarch - -BuildRequires: python3-fedmsg -BuildRequires: help2man -BuildRequires: python3-kobo -BuildRequires: python3-kobo-rpmlib -BuildRequires: python3-devel -BuildRequires: python3-flask-migrate -BuildRequires: python3-pdc-client -BuildRequires: python3-flask-script -BuildRequires: python3-httplib2 -BuildRequires: python3-munch -BuildRequires: python3-odcs-client -BuildRequires: python3-dogpile-cache -BuildRequires: python3-krbcontext -BuildRequires: python3-flask-login -BuildRequires: python3-ldap -BuildRequires: python3-pyyaml -BuildRequires: git -BuildRequires: python3-qpid-proton -BuildRequires: python3-tabulate -BuildRequires: python3-kerberos - -BuildRequires: python3-setuptools -BuildRequires: python3-fedora -BuildRequires: python3-flask -BuildRequires: python3-flask-sqlalchemy -BuildRequires: python3-mock -BuildRequires: python3-psutil -BuildRequires: python3-pytest -BuildRequires: python3-pyOpenSSL -BuildRequires: python3-sqlalchemy -BuildRequires: python3-koji -BuildRequires: python3-psycopg2 -BuildRequires: python3-defusedxml -BuildRequires: python3-prometheus_client - -BuildRequires: systemd -%{?systemd_requires} - -Requires: python3-fedmsg -Requires: systemd -Requires: python3-kobo -Requires: python3-kobo-rpmlib -Requires: python3-openidc-client -Requires: python3-pdc-client -Requires: python3-flask-script -Requires: python3-httplib2 -Requires: python3-munch -Requires: python3-odcs-client -Requires: python3-dogpile-cache -Requires: python3-krbcontext -Requires: python3-flask-login -Requires: python3-ldap -Requires: python3-pyyaml -Requires: git -Requires: python3-qpid-proton -Requires: python3-tabulate - -Requires: python3-koji -Requires: python3-fedora -Requires: python3-flask -Requires: python3-flask-migrate -Requires: python3-flask-sqlalchemy -Requires: python3-mock -Requires: python3-psutil -Requires: python3-pyOpenSSL -Requires: python3-sqlalchemy -Requires: python3-systemd -Requires: python3-psycopg2 -Requires: python3-defusedxml -Requires: python3-prometheus_client -Requires: python3-kerberos - - -%description -Freshmaker is a service scheduling rebuilds of artifacts as new content becomes available. - - -%prep -%setup -q - -sed -i '/ldap/d' requirements.txt -sed -i '/koji/d' requirements.txt - - -%build -%py3_build - - -%install -%py3_install - -export PYTHONPATH=%{buildroot}%{python3_sitelib} -mkdir -p %{buildroot}%{_mandir}/man1 -for command in freshmaker-manager freshmaker-frontend freshmaker-gencert freshmaker-upgradedb ; do -FRESHMAKER_CONFIG_FILE=conf/config.py %{buildroot}%{_bindir}/$command --help || true -FRESHMAKER_CONFIG_FILE=conf/config.py help2man -N --version-string=%{version} \ - %{buildroot}%{_bindir}/$command > \ - %{buildroot}%{_mandir}/man1/$command.1 -done - -install -d -m 0755 %{buildroot}%{_datadir}/freshmaker -install -p -m 0644 contrib/freshmaker.wsgi %{buildroot}%{_datadir}/freshmaker - - -%files -%doc README.md -%license LICENSE -%{python3_sitelib}/freshmaker* -%{_bindir}/freshmaker-* -%{_mandir}/man1/freshmaker-*.1* -%dir %{_sysconfdir}/freshmaker -%{_datadir}/freshmaker -%{_sysconfdir}/fedmsg.d/* -%config(noreplace) %{_sysconfdir}/freshmaker/config.py -%exclude %{_sysconfdir}/freshmaker/*.py[co] -%exclude %{_sysconfdir}/fedmsg.d/*.py[co] -%exclude %{python3_sitelib}/conf/ - - -%changelog diff --git a/.copr/prepare_spec.sh b/.copr/prepare_spec.sh deleted file mode 100644 index 18f8144..0000000 --- a/.copr/prepare_spec.sh +++ /dev/null @@ -1,5 +0,0 @@ -sudo dnf -y install python git fedpkg python-setuptools -FRESHMAKER_VERSION=$(python setup.py -V) -FRESHMAKER_RELEASE=$(git log -1 --pretty=format:%ct) -sed -e "s|\$FRESHMAKER_VERSION|$FRESHMAKER_VERSION|g" \ - -e "s|\$FRESHMAKER_RELEASE|$FRESHMAKER_RELEASE|g" ./.copr/freshmaker.spec.in > ./.copr/freshmaker.spec; diff --git a/Dockerfile b/Dockerfile index 9bafc24..547a8f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,52 @@ +# TODO: Upgrade to a more recent fedora version FROM fedora:29 + +# The caller should build a Freshmaker RPM package and then pass it in this arg. +ARG cacert_url=undefined +ARG appversion=undefined + LABEL \ name="Freshmaker application" \ vendor="Freshmaker developers" \ license="GPLv2+" \ - build-date="" -# The caller should build a Freshmaker RPM package and then pass it in this arg. -ARG freshmaker_rpm -ARG cacert_url=undefined -COPY $freshmaker_rpm /tmp - -RUN cd /etc/yum.repos.d/ \ - && dnf -v -y install 'dnf-command(config-manager)' \ - && dnf config-manager --add-repo http://download-ipv4.eng.brq.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-F-29/compose/Everything/x86_64/os/ \ - && dnf -y clean all \ - && dnf -v --nogpg -y install \ - httpd python3-mod_wsgi mod_auth_gssapi python3-rhmsg mod_ssl python3-odcs-client \ - /tmp/$(basename $freshmaker_rpm) \ - && dnf -y -v downgrade https://kojipkgs.fedoraproject.org//packages/qpid-proton/0.26.0/1.fc29/x86_64/qpid-proton-c-0.26.0-1.fc29.x86_64.rpm \ - https://kojipkgs.fedoraproject.org//packages/qpid-proton/0.26.0/1.fc29/x86_64/python3-qpid-proton-0.26.0-1.fc29.x86_64.rpm \ - && dnf -y -v upgrade https://kojipkgs.fedoraproject.org/packages/kobo/0.10.0/1.fc31/noarch/python3-kobo-0.10.0-1.fc31.noarch.rpm \ - https://kojipkgs.fedoraproject.org/packages/kobo/0.10.0/1.fc31/noarch/python3-kobo-rpmlib-0.10.0-1.fc31.noarch.rpm \ - && dnf -y clean all \ - && rm -f /tmp/* + build-date="" \ + version="$appversion" + + +# An internal yum repo is needed for rhmsg +ADD http://download.devel.redhat.com/rel-eng/RCMTOOLS/rcm-tools-fedora.repo /etc/yum.repos.d/ +# ...but the image doesn't have the required root CA installed +RUN sed -i 's_https://_http://_' /etc/yum.repos.d/rcm-tools-fedora.repo + +COPY yum-packages.txt /tmp/yum-packages.txt + +RUN \ + dnf -y install $(cat /tmp/yum-packages.txt) python3-rhmsg && \ + dnf -y downgrade \ + https://kojipkgs.fedoraproject.org//packages/qpid-proton/0.26.0/1.fc29/x86_64/qpid-proton-c-0.26.0-1.fc29.x86_64.rpm \ + https://kojipkgs.fedoraproject.org//packages/qpid-proton/0.26.0/1.fc29/x86_64/python3-qpid-proton-0.26.0-1.fc29.x86_64.rpm \ + && \ + dnf -y upgrade https://kojipkgs.fedoraproject.org/packages/kobo/0.10.0/1.fc31/noarch/python3-kobo-0.10.0-1.fc31.noarch.rpm \ + https://kojipkgs.fedoraproject.org/packages/kobo/0.10.0/1.fc31/noarch/python3-kobo-rpmlib-0.10.0-1.fc31.noarch.rpm \ + && \ + dnf clean all + +WORKDIR /src + +COPY . . + +RUN \ + # All dependencies should've been installed from RPMs + echo '' > requirements.txt && \ + pip3 install . --no-deps + +RUN mkdir -p /usr/share/freshmaker && cp contrib/freshmaker.wsgi /usr/share/freshmaker/ + +RUN \ + FRESHMAKER_CONFIG_FILE=/etc/freshmaker/config.py FRESHMAKER_CONFIG_SECTION=DevConfiguration freshmaker-manager --help &&\ + FRESHMAKER_CONFIG_FILE=/etc/freshmaker/config.py FRESHMAKER_CONFIG_SECTION=DevConfiguration freshmaker-frontend --help &&\ + FRESHMAKER_CONFIG_FILE=/etc/freshmaker/config.py FRESHMAKER_CONFIG_SECTION=DevConfiguration freshmaker-gencert --help &&\ + FRESHMAKER_CONFIG_FILE=/etc/freshmaker/config.py FRESHMAKER_CONFIG_SECTION=DevConfiguration freshmaker-upgradedb --help RUN if [ "$cacert_url" != "undefined" ]; then \ cd /etc/pki/ca-trust/source/anchors \ diff --git a/Jenkinsfile b/Jenkinsfile index 76bd9b3..8f54561 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,6 +5,8 @@ import groovy.json.* // 'global' var to store git info def scmVars +// 'global' var to store application version +def appversion try { // massive try{} catch{} around the entire build for failure notifications @@ -45,33 +47,20 @@ node('master'){ node('fedora-29') { stage('Prepare') { checkout scm - sh 'sudo rm -f rpmbuild-output/*.src.rpm' - sh 'mkdir -p rpmbuild-output' - sh 'make -f .copr/Makefile srpm outdir=./rpmbuild-output/' - sh 'sudo dnf -y builddep ./rpmbuild-output/freshmaker-*.src.rpm' - // TODO: some of the deps here should probably be in BuildRequires - sh 'sudo dnf -y install \ - gcc \ - krb5-devel \ - openldap-devel \ - python3-sphinxcontrib-httpdomain python3-pytest-cov \ - python3-flake8 python3-pylint python3-sphinx \ - python3-dogpile-cache \ - python3-fedmsg \ - python3-flask \ - python3-prometheus_client \ - python3-PyYAML \ - python3-requests \ - python3-flask-login \ - python3-flask-sqlalchemy \ - python3-ldap \ - python3-kobo \ - python3-kobo-rpmlib \ - python3-defusedxml \ - python3-rpm \ - python3-tox' - /* Needed to get the latest mock configs */ - sh 'sudo dnf -y update mock-core-configs' + // Install requirements for testing harness + sh 'sudo dnf -y install python3-tox python3-flake8 `cat yum-packages.txt`' + // The fedora-29 node has an older version of setuptools that causes + // tox to fail processing the dependencies + sh 'sudo dnf update -y python3-setuptools' + } + stage('Determine Version') { + // TODO: Set appversion as a label on the image? + appversion = sh(returnStdout: true, script: """ + printf `python setup.py -V`-`git log -1 --pretty=format:%ct` + """).trim() + } + stage('Run unit tests') { + sh 'tox' } stage('Build Docs') { sh ''' @@ -80,7 +69,7 @@ node('fedora-29') { python3-sphinxcontrib-httpdomain \ python3-sphinxcontrib-issuetracker ''' - sh 'FRESHMAKER_DEVELOPER_ENV=1 make -C docs html' + sh 'tox -e docs' archiveArtifacts artifacts: 'docs/_build/html/**' } if (scmVars.GIT_BRANCH == 'origin/master') { @@ -108,50 +97,21 @@ node('fedora-29') { } } } - stage('Run unit tests') { - sh 'tox' - } - /* We take a flock on the mock configs, to avoid multiple unrelated jobs on - * the same Jenkins slave trying to use the same mock root at the same - * time, which will error out. */ - stage('Build RPM') { - parallel ( - 'F29': { - sh """ - mkdir -p mock-result/f29 - flock /etc/mock/fedora-29-x86_64.cfg \ - /usr/bin/mock -v --enable-network --resultdir=mock-result/f29 -r fedora-29-x86_64 --clean --rebuild rpmbuild-output/*.src.rpm - """ - archiveArtifacts artifacts: 'mock-result/f29/**' - }, - ) - } } if ("${env.JOB_NAME}" != 'freshmaker-prs') { node('docker') { stage('Build Docker container') { checkout scm - // Remember to reflect the version change in the Dockerfile in the future. - sh 'grep -q "FROM fedora:29" Dockerfile' - unarchive mapping: ['mock-result/f29/': '.'] - def f29_rpm = findFiles(glob: 'mock-result/f29/**/*.noarch.rpm')[0] - def appversion = sh(returnStdout: true, script: """ - rpm2cpio ${f29_rpm} | \ - cpio --quiet --extract --to-stdout ./usr/lib/python\\*/site-packages/freshmaker\\*.egg-info/PKG-INFO | \ - awk '/^Version: / {print \$2}' - """).trim() - /* Git builds will have a version like 0.3.2.dev1+git.3abbb08 following - * the rules in PEP440. But Docker does not let us have + in the tag - * name, so let's munge it here. */ - appversion = appversion.replace('+', '-') sh 'docker image prune -a -f' + // Remove non source files so they don't end up in the image + sh 'git clean -fdx && rm -rf .git' docker.withRegistry( 'https://docker-registry.upshift.redhat.com/', 'factory2-upshift-registry-token') { /* Note that the docker.build step has some magic to guess the * Dockerfile used, which will break if the build directory (here ".") * is not the final argument in the string. */ - def image = docker.build "factory2/freshmaker:internal-${appversion}", "--build-arg freshmaker_rpm=$f29_rpm --build-arg cacert_url=https://password.corp.redhat.com/RH-IT-Root-CA.crt ." + def image = docker.build "factory2/freshmaker:internal-${appversion}", "--build-arg cacert_url=https://password.corp.redhat.com/RH-IT-Root-CA.crt --build-arg appversion=${appversion} ." /* Pushes to the internal registry can sometimes randomly fail * with "unknown blob" due to a known issue with the registry * storage configuration. So we retry up to 3 times. */ @@ -163,20 +123,15 @@ node('docker') { docker.withRegistry( 'https://quay.io/', 'quay-io-factory2-builder-sa-credentials') { - def image = docker.build "factory2/freshmaker:${appversion}", "--build-arg freshmaker_rpm=$f29_rpm ." + def image = docker.build "factory2/freshmaker:${appversion}", " --build-arg appversion=${appversion} ." image.push() } - /* Save container version for later steps (this is ugly but I can't find anything better...) */ - writeFile file: 'appversion', text: appversion - archiveArtifacts artifacts: 'appversion' } } node('docker') { if (scmVars.GIT_BRANCH == 'origin/master') { stage('Tag "latest".') { checkout scm - unarchive mapping: ['appversion': 'appversion'] - def appversion = readFile('appversion').trim() docker.withRegistry( 'https://docker-registry.upshift.redhat.com/', 'factory2-upshift-registry-token') { diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index dc29384..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -W -SPHINXBUILD = sphinx-build-3 -SPHINXPROJ = Freshmaker -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/tox.ini b/tox.ini index cef5396..8a43a06 100644 --- a/tox.ini +++ b/tox.ini @@ -42,6 +42,12 @@ ignore_outcome = True ignore = E501,E731,W504 exclude = dev_scripts/*,freshmaker/migrations/*,.tox/*,build/*,__pycache__,scripts/print_handlers_md.py,.copr/*,.env +[testenv:docs] +description = invoke sphinx-build to build the HTML docs +basepython = python3 +commands = + python3 -m 'sphinx' {toxinidir}/docs "{toxinidir}/docs/_build/html" -W {posargs} + [pytest] addopts = --cov=freshmaker diff --git a/yum-packages.txt b/yum-packages.txt new file mode 100644 index 0000000..992467e --- /dev/null +++ b/yum-packages.txt @@ -0,0 +1,40 @@ +git +httpd +mod_auth_gssapi +mod_ssl +python3-defusedxml +python3-dogpile-cache +python3-fedmsg +python3-fedora +python3-flask +python3-flask-login +python3-flask-migrate +python3-flask-script +python3-flask-sqlalchemy +python3-httplib2 +python3-kerberos +python3-kobo +python3-kobo-rpmlib +python3-koji +python3-krbcontext +python3-ldap +python3-mock +python3-mod_wsgi +python3-munch +python3-odcs-client +python3-openidc-client +python3-pdc-client +python3-prometheus_client +python3-psutil +python3-psycopg2 +python3-pyOpenSSL +python3-pyyaml +python3-qpid-proton +python3-requests +python3-rpm +python3-setuptools +python3-sqlalchemy +python3-systemd +python3-tabulate +systemd +