From a7de1cca425aec70ce1636c484754d80c000689f Mon Sep 17 00:00:00 2001 From: mprahl Date: Jan 31 2019 16:21:27 +0000 Subject: Install from source when building the container image instead of needing to supply an RPM Since the RPM is only used to build the container image, it adds an unnecessary layer of complexity in the Jenkins job. This is an attempt to simplify the Greenwave build process. --- diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e8c3ace --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.env/ +.git/ +.pytest_cache/ +.tox/ +.vscode/ +build/ +dist/ +htmlcov/ +*.egg-info/ diff --git a/Dockerfile b/Dockerfile index a5973d1..9e1d2fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,16 +5,19 @@ LABEL \ license="GPLv2+" \ build-date="" -# The caller should build a greenwave RPM package using ./rpmbuild.sh and then pass it in this arg. -ARG greenwave_rpm # The caller can optionally provide a cacert url ARG cacert_url=undefined -COPY $greenwave_rpm /tmp +WORKDIR /src RUN dnf -y install \ + python3-dogpile-cache \ + python3-fedmsg \ + python3-flask \ python3-gunicorn \ python3-memcached \ - /tmp/$(basename $greenwave_rpm) \ + python3-prometheus_client \ + python3-PyYAML \ + python3-requests \ && dnf -y clean all \ && rm -rf /tmp/* @@ -25,7 +28,10 @@ RUN if [ "$cacert_url" != "undefined" ]; then \ fi # This will allow a non-root user to install a custom root CA at run-time RUN chmod 777 /etc/pki/tls/certs/ca-bundle.crt -COPY docker/install-ca.sh /opt +COPY . . +RUN pip3 install . --no-deps +# Remove the default fedmsg config files included in the repo +RUN rm -rf ./fedmsg.d USER 1001 EXPOSE 8080 -ENTRYPOINT /opt/install-ca.sh && gunicorn-3 --workers 8 --bind 0.0.0.0:8080 --access-logfile=- --enable-stdio-inheritance greenwave.wsgi:app +ENTRYPOINT docker/install-ca.sh && gunicorn-3 --workers 8 --bind 0.0.0.0:8080 --access-logfile=- --enable-stdio-inheritance greenwave.wsgi:app diff --git a/Jenkinsfile b/Jenkinsfile index 4bb55c5..e2fdab5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,6 +6,17 @@ import groovy.json.* // 'global' var to store git info def scmVars +// Greenwave RPM dependencies +def installDepsCmd = ''' +sudo dnf -y install \ + python3-dogpile-cache \ + python3-fedmsg \ + python3-flask \ + python3-prometheus_client \ + python3-PyYAML \ + python3-requests +'''.trim() + try { // massive try{} catch{} around the entire build for failure notifications node('master'){ @@ -47,11 +58,11 @@ node('fedora-28') { returnStdout: true ).trim() - sh ''' - sudo dnf -y builddep greenwave.spec + sh """ + ${installDepsCmd} sudo dnf -y install python3-flake8 python3-pylint python3-sphinx \ python3-sphinxcontrib-httpdomain python3-pytest-cov - ''' + """ /* Needed to get the latest /etc/mock/fedora-28-x86_64.cfg */ sh 'sudo dnf -y update mock-core-configs' stage('Invoke Flake8') { @@ -61,7 +72,6 @@ node('fedora-28') { sh 'pylint-3 --reports=n greenwave' } stage('Run unit tests') { - // Yes, this is also done while building the RPM, but we need coverage sh ''' rm -rf htmlcov coverage.xml pytest-3 greenwave/tests/ \ @@ -82,6 +92,12 @@ node('fedora-28') { ]) } stage('Build Docs') { + sh ''' + sudo dnf install -y \ + python3-sphinx \ + python3-sphinxcontrib-httpdomain \ + python3-sphinxcontrib-issuetracker + ''' sh 'DEV=true GREENWAVE_CONFIG=$(pwd)/conf/settings.py.example make -C docs html' archiveArtifacts artifacts: 'docs/_build/html/**' } @@ -110,43 +126,16 @@ node('fedora-28') { } } } - stage('Build SRPM') { - sh './rpmbuild.sh -bs' - archiveArtifacts artifacts: 'rpmbuild-output/**' - } - /* We take a flock on the mock configs, to avoid multiple unrelated jobs on - * the same Jenkins slave trying to use the same mock root at the same - * time, which will error out. */ - stage('Build RPM') { - parallel ( - 'F28': { - sh """ - mkdir -p mock-result/f28 - flock /etc/mock/fedora-28-x86_64.cfg \ - /usr/bin/mock -v --enable-network --resultdir=mock-result/f28 -r fedora-28-x86_64 --clean --rebuild rpmbuild-output/*.src.rpm - """ - archiveArtifacts artifacts: 'mock-result/f28/**' - }, - ) - } - stage('Invoke Rpmlint') { - parallel ( - 'F28': { - sh 'rpmlint -f rpmlint-config.py mock-result/f28/*.rpm' - }, - ) - } } node('docker') { checkout scm stage('Build Docker container') { - unarchive mapping: ['mock-result/f28/': '.'] - def f28_rpm = findFiles(glob: 'mock-result/f28/**/*.noarch.rpm')[0] - def appversion = sh(returnStdout: true, script: """ - rpm2cpio ${f28_rpm} | \ - cpio --quiet --extract --to-stdout ./usr/lib/python3\\*/site-packages/greenwave\\*.egg-info/PKG-INFO | \ - awk '/^Version: / {print \$2}' - """).trim() + def appversion = sh(returnStdout: true, script: './get-version.sh').trim() + // Set the derived version in __init__.py + sh """ + sed --regexp-extended --in-place \ + -e "/^__version__ = /c\\__version__ = '${appversion}'" greenwave/__init__.py + """.trim() /* Git builds will have a version like 0.3.2.dev1+git.3abbb08 following * the rules in PEP440. But Docker does not let us have + in the tag * name, so let's munge it here. */ @@ -157,7 +146,7 @@ node('docker') { /* Note that the docker.build step has some magic to guess the * Dockerfile used, which will break if the build directory (here ".") * is not the final argument in the string. */ - def image = docker.build "factory2/greenwave:internal-${appversion}", "--build-arg greenwave_rpm=$f28_rpm --build-arg cacert_url=https://password.corp.redhat.com/RH-IT-Root-CA.crt ." + def image = docker.build "factory2/greenwave:internal-${appversion}", "--build-arg cacert_url=https://password.corp.redhat.com/RH-IT-Root-CA.crt ." /* Pushes to the internal registry can sometimes randomly fail * with "unknown blob" due to a known issue with the registry * storage configuration. So we retry up to 3 times. */ @@ -169,7 +158,7 @@ node('docker') { docker.withRegistry( 'https://quay.io/', 'quay-io-factory2-builder-sa-credentials') { - def image = docker.build "factory2/greenwave:${appversion}", "--build-arg greenwave_rpm=$f28_rpm ." + def image = docker.build "factory2/greenwave:${appversion}", "." image.push() } /* Save container version for later steps (this is ugly but I can't find anything better...) */ @@ -186,7 +175,7 @@ node('fedora-28') { /* Also need to install Greenwave's dependencies, since we are running it * locally not in Openshift for now. */ - sh 'sudo dnf -y builddep greenwave.spec' + sh installDepsCmd def openshiftHost = 'greenwave-test.cloud.paas.upshift.redhat.com' def buildTag = "${env.BUILD_TAG}".replace('jenkins-','') diff --git a/get-version.sh b/get-version.sh new file mode 100755 index 0000000..08cc69f --- /dev/null +++ b/get-version.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0+ + +# Prints the current version based on the current git revision. + +set -e + +name=greenwave +if [ "$(git tag | wc -l)" -eq 0 ] ; then + # never been tagged since the project is just starting out + lastversion="0.0" + revbase="" +else + lasttag="$(git describe --abbrev=0 HEAD)" + lastversion="${lasttag##${name}-}" + revbase="^$lasttag" +fi +if [ "$(git rev-list $revbase HEAD | wc -l)" -eq 0 ] ; then + # building a tag + version="$lastversion" +else + # git builds count as a pre-release of the next version + version="$lastversion" + version="${version%%[a-z]*}" # strip non-numeric suffixes like "rc1" + # increment the last portion of the version + version="${version%.*}.$((${version##*.} + 1))" + commitcount=$(git rev-list $revbase HEAD | wc -l) + commitsha=$(git rev-parse --short HEAD) + version="${version}.dev${commitcount}+git.${commitsha}" +fi + +echo $version diff --git a/greenwave.spec b/greenwave.spec deleted file mode 100644 index e883d29..0000000 --- a/greenwave.spec +++ /dev/null @@ -1,58 +0,0 @@ - -%global upstream_version 0.9.13 - -Name: greenwave -Version: 0.9.13 -Release: 1%{?dist} -Summary: Service for gating on automated tests -License: GPLv2+ -URL: https://pagure.io/greenwave -Source0: https://files.pythonhosted.org/packages/source/g/%{name}/%{name}-%{upstream_version}.tar.gz - -BuildRequires: python3-devel -BuildRequires: python3-setuptools -BuildRequires: python3-sphinx -BuildRequires: python3-sphinxcontrib-httpdomain -BuildRequires: python3-sphinxcontrib-issuetracker -BuildRequires: python3-flask -BuildRequires: python3-pytest -BuildRequires: python3-requests -BuildRequires: python3-PyYAML -BuildRequires: python3-dogpile-cache -BuildRequires: python3-fedmsg -BuildRequires: python3-prometheus_client -BuildArch: noarch -Requires: python3-flask -Requires: python3-requests -Requires: python3-PyYAML -Requires: python3-dogpile-cache -Requires: python3-fedmsg -Requires: python3-prometheus_client - -%description -Greenwave is a service for gating on automated tests by querying ResultsDB and -WaiverDB. - -%prep -%setup -q -n %{name}-%{upstream_version} - -%build -%py3_build -DEV=true GREENWAVE_CONFIG=$(pwd)/conf/settings.py.example make -C docs SPHINXOPTS= html text - -%install -%py3_install - -%check -export PYTHONPATH=%{buildroot}/%{python3_sitelib} -py.test-3 greenwave/tests/ - -%files -%license COPYING -%doc README.md conf -%doc docs/_build/html docs/_build/text -%{python3_sitelib}/%{name} -%{python3_sitelib}/%{name}*.egg-info -%{_sysconfdir}/fedmsg.d/* - -%changelog diff --git a/rpmbuild.sh b/rpmbuild.sh deleted file mode 100755 index 9dabcb9..0000000 --- a/rpmbuild.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# SPDX-License-Identifier: GPL-2.0+ - -# Builds a development (S)RPM from the current git revision. - -set -e - -if [ $# -eq 0 ] ; then - echo "Usage: $1 -bs|-bb " >&2 - echo "Hint: -bs builds SRPM, -bb builds RPM, refer to rpmbuild(8)" >&2 - exit 1 -fi - -name=greenwave -if [ "$(git tag | wc -l)" -eq 0 ] ; then - # never been tagged since the project is just starting out - lastversion="0.0" - revbase="" -else - lasttag="$(git describe --abbrev=0 HEAD)" - lastversion="${lasttag##${name}-}" - revbase="^$lasttag" -fi -if [ "$(git rev-list $revbase HEAD | wc -l)" -eq 0 ] ; then - # building a tag - rpmver="" - rpmrel="" - version="$lastversion" -else - # git builds count as a pre-release of the next version - version="$lastversion" - version="${version%%[a-z]*}" # strip non-numeric suffixes like "rc1" - # increment the last portion of the version - version="${version%.*}.$((${version##*.} + 1))" - commitcount=$(git rev-list $revbase HEAD | wc -l) - commitsha=$(git rev-parse --short HEAD) - rpmver="${version}" - rpmrel="0.git.${commitcount}.${commitsha}" - version="${version}.dev${commitcount}+git.${commitsha}" -fi - -workdir="$(mktemp -d)" -trap "rm -rf $workdir" EXIT -outdir="$(readlink -f ./rpmbuild-output)" -mkdir -p "$outdir" - -git archive --format=tar HEAD | tar -C "$workdir" -xf - -if [ -n "$rpmrel" ] ; then - # need to hack the version in the spec - sed --regexp-extended --in-place \ - -e "/%global upstream_version /c\%global upstream_version ${version}" \ - -e "/^Version:/cVersion: ${rpmver}" \ - -e "/^Release:/cRelease: ${rpmrel}%{?dist}" \ - "$workdir/${name}.spec" - # also hack the Python module version - sed --regexp-extended --in-place \ - -e "/^__version__ = /c\\__version__ = '$version'" \ - "$workdir/greenwave/__init__.py" -fi -( cd "$workdir" && python setup.py sdist ) -mv "$workdir"/dist/*.tar.gz "$workdir" - -rpmbuild \ - --define "_topdir $workdir" \ - --define "_sourcedir $workdir" \ - --define "_specdir $workdir" \ - --define "_rpmdir $outdir" \ - --define "_srcrpmdir $outdir" \ - "$@" "$workdir/${name}.spec"