From 6595a07cfc9d6df4267bc0eadbc340d08d4844b9 Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Nov 28 2018 09:04:13 +0000 Subject: [PATCH 1/9] Make code Python 3 compatible --- diff --git a/execdb/__init__.py b/execdb/__init__.py index 848ff62..74a34dd 100644 --- a/execdb/__init__.py +++ b/execdb/__init__.py @@ -18,8 +18,8 @@ # Josef Skladanka from flask import Flask, render_template -from flask.ext.login import LoginManager -from flask.ext.sqlalchemy import SQLAlchemy +from flask_login import LoginManager +from flask_sqlalchemy import SQLAlchemy import logging import os @@ -69,7 +69,7 @@ def setup_logging(): root_logger.setLevel(logging.DEBUG) if app.config['STREAM_LOGGING']: - print "doing stream logging" + print("doing stream logging") stream_handler = logging.StreamHandler() stream_handler.setLevel(loglevel) stream_handler.setFormatter(formatter) @@ -77,7 +77,7 @@ def setup_logging(): app.logger.addHandler(stream_handler) if app.config['SYSLOG_LOGGING']: - print "doing syslog logging" + print("doing syslog logging") syslog_handler = logging.handlers.SysLogHandler( address='/dev/log', facility=logging.handlers.SysLogHandler.LOG_LOCAL4) @@ -87,7 +87,7 @@ def setup_logging(): app.logger.addHandler(syslog_handler) if app.config['FILE_LOGGING'] and app.config['LOGFILE']: - print "doing file logging to %s" % app.config['LOGFILE'] + print("doing file logging to %s" % app.config['LOGFILE']) file_handler = logging.handlers.RotatingFileHandler( app.config['LOGFILE'], maxBytes=500000, diff --git a/execdb/cli.py b/execdb/cli.py index 8e1e1ca..5ad8771 100644 --- a/execdb/cli.py +++ b/execdb/cli.py @@ -50,7 +50,7 @@ def get_alembic_config(): def upgrade_db(*args): - print "Upgrading Database to Latest Revision" + print("Upgrading Database to Latest Revision") alembic_cfg = get_alembic_config() al_command.upgrade(alembic_cfg, "head") @@ -64,20 +64,20 @@ def init_alembic(*args): current_rev = context.get_current_revision() if not current_rev: - print "Initializing alembic" - print " - Setting the current version to the first revision" + print("Initializing alembic") + print(" - Setting the current version to the first revision") al_command.stamp(alembic_cfg, "1cefaba53e0") else: - print "Alembic already initialized" + print("Alembic already initialized") def initialize_db(destructive): alembic_cfg = get_alembic_config() - print "Initializing database" + print("Initializing database") if destructive: - print " - Dropping all tables" + print(" - Dropping all tables") db.drop_all() # check whether the table 'job' exists @@ -85,9 +85,9 @@ def initialize_db(destructive): insp = reflection.Inspector.from_engine(db.engine) table_names = insp.get_table_names() if 'job' not in table_names and 'Job' not in table_names: - print " - Creating tables" + print(" - Creating tables") db.create_all() - print " - Stamping alembic's current version to 'head'" + print(" - Stamping alembic's current version to 'head'") al_command.stamp(alembic_cfg, "head") # check to see if the db has already been initialized by checking for an @@ -95,18 +95,18 @@ def initialize_db(destructive): context = MigrationContext.configure(db.engine.connect()) current_rev = context.get_current_revision() if current_rev: - print " - Database is currently at rev %s" % current_rev + print(" - Database is currently at rev %s" % current_rev) upgrade_db(destructive) else: - print "WARN: You need to have your db stamped with an alembic revision" - print " Run 'init_alembic' sub-command first." + print("WARN: You need to have your db stamped with an alembic revision") + print(" Run 'init_alembic' sub-command first.") def mock_data(destructive): - print "Populating tables with mock-data" + print("Populating tables with mock-data") if destructive or not db.session.query(User).count(): - print " - User" + print(" - User") data_users = [('admin', 'admin'), ('user', 'user')] for d in data_users: @@ -115,7 +115,7 @@ def mock_data(destructive): db.session.commit() else: - print " - skipped User" + print(" - skipped User") def mock_data_live(destructive): import time @@ -168,9 +168,9 @@ def main(): (options, args) = parser.parse_args() if len(args) != 1 or args[0] not in possible_commands: - print usage - print - print 'Please use one of the following commands: %s' % str(possible_commands) + print(usage) + print("\n") + print('Please use one of the following commands: %s' % str(possible_commands)) sys.exit(1) command = { @@ -181,8 +181,8 @@ def main(): 'init_alembic': init_alembic, }[args[0]] if not options.destructive: - print "Proceeding in non-destructive mode. To perform destructive "\ - "steps use -d option." + print("Proceeding in non-destructive mode. To perform destructive "\ + "steps use -d option.") command(options.destructive) diff --git a/execdb/controllers/admin.py b/execdb/controllers/admin.py index 6a97990..f665a37 100644 --- a/execdb/controllers/admin.py +++ b/execdb/controllers/admin.py @@ -18,7 +18,7 @@ # Josef Skladanka from flask import Blueprint, render_template, flash, url_for -from flask.ext.login import login_required +from flask_login import login_required admin = Blueprint('admin', __name__) diff --git a/execdb/controllers/login_page.py b/execdb/controllers/login_page.py index 8e3b2a1..6ec24b4 100644 --- a/execdb/controllers/login_page.py +++ b/execdb/controllers/login_page.py @@ -18,10 +18,10 @@ # Josef Skladanka from flask import Blueprint, render_template, redirect, flash, url_for, request -from flask.ext.wtf import Form +from flask_wtf import Form from wtforms import TextField, PasswordField, HiddenField, RadioField from wtforms.validators import Required -from flask.ext.login import login_user, logout_user, login_required, current_user, AnonymousUserMixin +from flask_login import login_user, logout_user, login_required, current_user, AnonymousUserMixin from execdb import app, login_manager diff --git a/execdb/controllers/main.py b/execdb/controllers/main.py index f4abcde..623e888 100644 --- a/execdb/controllers/main.py +++ b/execdb/controllers/main.py @@ -21,7 +21,7 @@ from flask import Blueprint, render_template, request, jsonify import werkzeug.exceptions from sqlalchemy.orm import exc as orm_exc -from flask.ext.restful import reqparse +from flask_restful import reqparse from werkzeug.exceptions import HTTPException from werkzeug.exceptions import BadRequest as JSONBadRequest diff --git a/execdb/models/user.py b/execdb/models/user.py index e80a347..a5baf16 100644 --- a/execdb/models/user.py +++ b/execdb/models/user.py @@ -18,7 +18,7 @@ # Josef Skladanka from execdb import db -from flask.ext.login import UserMixin +from flask_login import UserMixin from werkzeug.security import generate_password_hash, check_password_hash diff --git a/run_cli.py b/run_cli.py index 5344eaf..5402245 100644 --- a/run_cli.py +++ b/run_cli.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014, Red Hat, Inc # diff --git a/runapp.py b/runapp.py index 316313f..45964af 100644 --- a/runapp.py +++ b/runapp.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # runapp.py - script to facilitate running the execdb app from the CLI # From 345b3feeec4d15d828eedce7bd643a5f14f4ced1 Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Nov 28 2018 09:05:08 +0000 Subject: [PATCH 2/9] Bump to 0.0.11 --- diff --git a/execdb.spec b/execdb.spec index 2bc4755..250e37c 100644 --- a/execdb.spec +++ b/execdb.spec @@ -1,6 +1,6 @@ Name: execdb # NOTE: if you update version, *make sure* to also update `execdb/__init__.py` -Version: 0.0.10 +Version: 0.0.11 Release: 1%{?dist} Summary: Execution status database for Taskotron @@ -10,24 +10,16 @@ Source0: https://qa.fedoraproject.org/releases/%{name}/%{name}-%{version} BuildArch: noarch -%if 0%{?fedora} <= 27 -Requires: python-alembic -Requires: python-flask -Requires: python-flask-sqlalchemy -Requires: python-flask-wtf -Requires: python-flask-login -%else -Requires: python2-alembic -Requires: python2-flask -Requires: python2-flask-sqlalchemy -Requires: python2-flask-wtf -Requires: python2-flask-login -%endif -Requires: python2-flask-restful -Requires: python2-six - -BuildRequires: python2-devel -BuildRequires: python2-setuptools +Requires: python3-alembic +Requires: python3-flask +Requires: python3-flask-sqlalchemy +Requires: python3-flask-wtf +Requires: python3-flask-login +Requires: python3-flask-restful +Requires: python3-six + +BuildRequires: python3-devel +BuildRequires: python3-setuptools %description ExecDB is a database that stores the execution status of jobs running @@ -43,10 +35,10 @@ started and finished, and some of their properties. rm -f %{buildroot}%{_sysconfdir}/execdb/*.py{c,o} %build -%py2_build +%py3_build %install -%py2_install +%py3_install # apache and wsgi settings install -d %{buildroot}%{_datadir}/execdb/conf @@ -64,16 +56,22 @@ install -p -m 0644 conf/settings.py.example %{buildroot}%{_sysconfdir}/execdb/se %files %doc README.md %license LICENSE -%{python2_sitelib}/execdb -%{python2_sitelib}/*.egg-info +%{python3_sitelib}/execdb +%{python3_sitelib}/*.egg-info + +%{_bindir}/execdb -%attr(755,root,root) %{_bindir}/execdb %dir %{_sysconfdir}/execdb %config(noreplace) %{_sysconfdir}/execdb/settings.py + %dir %{_datadir}/execdb %{_datadir}/execdb/* %changelog +* Wed Nov 28 2018 Frantisek Zatloukal - 0.0.11-1 +- Switch to Python 3 +- Drop Fedora 27 + * Fri Apr 27 2018 Frantisek Zatloukal - 0.0.10-1 - API - Fix show_job template to use right path for api diff --git a/execdb/__init__.py b/execdb/__init__.py index 74a34dd..405db7b 100644 --- a/execdb/__init__.py +++ b/execdb/__init__.py @@ -26,7 +26,7 @@ import os # the version as used in setup.py -__version__ = "0.0.10" +__version__ = "0.0.11" # Flask App diff --git a/init_db.sh b/init_db.sh index 4d371ce..da99bf8 100755 --- a/init_db.sh +++ b/init_db.sh @@ -2,7 +2,7 @@ # this is a simple script to aid in the setup of a new db # init db -python run_cli.py init_db ${@} +python3 run_cli.py init_db ${@} # insert mock data -python run_cli.py mock_data ${@} +python3 run_cli.py mock_data ${@} From f32197064b2bb0572e263d92b57bdbaf1840d50d Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Nov 28 2018 09:05:55 +0000 Subject: [PATCH 3/9] Makefile: Use generic Makefile provided by qa-make Project can be found here: https://pagure.io/fedora-qa/qa-make --- diff --git a/Makefile b/Makefile index c319993..28fde3b 100644 --- a/Makefile +++ b/Makefile @@ -1,35 +1,70 @@ -# -# Copyright 2013, Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -# general variables -VENV=test_env -SRC=execdb +# Copyright 2018, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +####################################################################### +# _____ _ _ _ _ _ # +# / ____| | | (_) | | | (_) # +# | | ___ _ __ | |_ _ __ _| |__ _ _| |_ _ _ __ __ _ # +# | | / _ \| '_ \| __| '__| | '_ \| | | | __| | '_ \ / _` | # +# | |___| (_) | | | | |_| | | | |_) | |_| | |_| | | | | (_| | # +# \_____\___/|_| |_|\__|_| |_|_.__/ \__,_|\__|_|_| |_|\__, | # +# __/ | # +# If you want to add/fix anything here, please create |___/ # +# PR at qa-make https://pagure.io/fedora-qa/qa-make # +# # +####################################################################### + +# Allows to print variables, eg. make print-SRC +print-% : ; @echo $* = $($*) + +# Get variables from Makefile.cfg +SRC=$(shell grep -s SRC Makefile.cfg | sed 's/SRC=//') +VENV=$(shell grep -s VENV Makefile.cfg | sed 's/VENV=//') +MODULENAME=$(shell grep -s MODULENAME Makefile.cfg | sed 's/MODULENAME=//') + +# Try to detect SRC in case we didn't find Makefile.cfg +ifeq ($(SRC),) +SRC=$(shell rpmspec -q --queryformat="%{NAME}\n" *.spec | head -1) +SPECNUM=$(shell ls -1 *.spec | wc -l) +ifneq ($(SPECNUM),1) +$(error Make sure you have either one spec file in the directory or configure it in Makefile.cfg) +endif +endif # Variables used for packaging SPECFILE=$(SRC).spec BASEARCH:=$(shell uname -i) DIST:=$(shell rpm --eval '%{dist}') -VERSION:=$(shell rpmspec -q --queryformat="%{VERSION}\n" $(SPECFILE) | uniq) -RELEASE:=$(subst $(DIST),,$(shell rpmspec -q --queryformat="%{RELEASE}\n" $(SPECFILE) | uniq)) +TARGETVER:=$(shell lsb_release -r |grep -o '[0-9]*') +TARGETDIST:=fc$(TARGETVER) +VERSION:=$(shell rpmspec -q --queryformat="%{VERSION}\n" $(SPECFILE) | head -1) +RELEASE:=$(shell rpmspec -q --queryformat="%{RELEASE}\n" $(SPECFILE) | head -1 | sed 's/$(DIST)/\.$(TARGETDIST)/g') NVR:=$(SRC)-$(VERSION)-$(RELEASE) GITBRANCH:=$(shell git rev-parse --abbrev-ref HEAD) -TARGETDIST:=fc25 -BUILDTARGET=fedora-25-x86_64 +BUILDTARGET:=fedora-$(TARGETVER)-x86_64 +KOJITARGET:=$(shell echo $(TARGETDIST) | sed 's/c//' | sed 's/el/epel-/') + +.PHONY: update-makefile +update-makefile: + curl --fail https://pagure.io/fedora-qa/qa-make/raw/master/f/Makefile -o Makefile.new + if ! cmp Makefile Makefile.new ; then mv Makefile.new Makefile ; fi + +.PHONY: test +.ONESHELL: test +test: $(VENV) + set -e + source $(VENV)/bin/activate; + TEST='true' py.test --cov-report=term-missing --cov $(MODULENAME); + deactivate + +.PHONY: test-ci +.ONESHELL: test-ci +test-ci: $(VENV) + set -e + source $(VENV)/bin/activate + TEST='true' py.test --cov-report=xml --cov $(MODULENAME) + deactivate .PHONY: pylint pylint: @@ -49,7 +84,7 @@ docs: .PHONY: clean clean: rm -rf dist - rm -rf execdb.egg-info + rm -rf $(SRC).egg-info rm -rf build rm -f pep8.out rm -f pylint.out @@ -60,20 +95,22 @@ archive: $(SRC)-$(VERSION).tar.gz .PHONY: $(SRC)-$(VERSION).tar.gz $(SRC)-$(VERSION).tar.gz: git archive $(GITBRANCH) --prefix=$(SRC)-$(VERSION)/ | gzip -c9 > $@ + mkdir -p build/$(VERSION)-$(RELEASE) + mv $(SRC)-$(VERSION).tar.gz build/$(VERSION)-$(RELEASE)/ -.PHONY: mocksrpm -mocksrpm: archive - mock -r $(BUILDTARGET) --buildsrpm --spec $(SPECFILE) --sources . - cp /var/lib/mock/$(BUILDTARGET)/result/$(NVR).$(TARGETDIST).src.rpm . +.PHONY: srpm +srpm: archive + mock -r $(BUILDTARGET) --buildsrpm --spec $(SPECFILE) --sources build/$(VERSION)-$(RELEASE)/ + cp /var/lib/mock/$(BUILDTARGET)/result/$(NVR).src.rpm build/$(VERSION)-$(RELEASE)/ -.PHONY: mockbuild -mockbuild: mocksrpm - mock -r $(BUILDTARGET) --no-clean --rebuild $(NVR).$(TARGETDIST).src.rpm - cp /var/lib/mock/$(BUILDTARGET)/result/$(NVR).$(TARGETDIST).noarch.rpm . +.PHONY: build +build: srpm + mock -r $(BUILDTARGET) --no-clean --rebuild build/$(VERSION)-$(RELEASE)/$(NVR).src.rpm + cp /var/lib/mock/$(BUILDTARGET)/result/*.rpm build/$(VERSION)-$(RELEASE)/ -#.PHONY: kojibuild -#kojibuild: mocksrpm -# koji build --scratch dist-6E-epel-testing-candidate $(NVR).$(TARGETDIST).src.rpm +.PHONY: scratch +scratch: srpm + koji build --scratch $(KOJITARGET) build/$(VERSION)-$(RELEASE)/$(NVR).src.rpm .PHONY: nvr nvr: @@ -87,7 +124,10 @@ cleanvenv: virtualenv: $(VENV) .PHONY: $(VENV) +.ONESHELL: $(VENV) $(VENV): - virtualenv $(VENV) - sh -c "set -e; . $(VENV)/bin/activate; pip install -r requirements.txt; \ - deactivate" + virtualenv --system-site-packages $(VENV) + set -e + source $(VENV)/bin/activate + pip install -r requirements.txt + deactivate From bfd961695ffd98b35c422181fd71f1c07354b060 Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Jan 22 2019 11:36:47 +0000 Subject: [PATCH 4/9] Fix FlaskWTFDeprecationWarning "flask_wtf.Form" has been renamed to "FlaskForm" and will be removed in 1.0. --- diff --git a/execdb/controllers/login_page.py b/execdb/controllers/login_page.py index 6ec24b4..508ee0f 100644 --- a/execdb/controllers/login_page.py +++ b/execdb/controllers/login_page.py @@ -18,7 +18,7 @@ # Josef Skladanka from flask import Blueprint, render_template, redirect, flash, url_for, request -from flask_wtf import Form +from flask_wtf import FlaskForm as Form from wtforms import TextField, PasswordField, HiddenField, RadioField from wtforms.validators import Required from flask_login import login_user, logout_user, login_required, current_user, AnonymousUserMixin From 5b510d3747184330de316ce7b35f143c3b438984 Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Jan 22 2019 12:14:45 +0000 Subject: [PATCH 5/9] Support also old-style flask_wtf import It needs to work also on F27 for a little longer, so add simple try/except to prefer new import, but fall back to the old one if needed --- diff --git a/execdb/controllers/login_page.py b/execdb/controllers/login_page.py index 508ee0f..3c84bec 100644 --- a/execdb/controllers/login_page.py +++ b/execdb/controllers/login_page.py @@ -18,7 +18,10 @@ # Josef Skladanka from flask import Blueprint, render_template, redirect, flash, url_for, request -from flask_wtf import FlaskForm as Form +try: + from flask_wtf import FlaskForm as Form +except ImportError: + from flask_wtf import Form from wtforms import TextField, PasswordField, HiddenField, RadioField from wtforms.validators import Required from flask_login import login_user, logout_user, login_required, current_user, AnonymousUserMixin From 7fcaeae3ec1f3ff4500b9a6af78e1a5753b8ae81 Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Jan 22 2019 14:54:33 +0000 Subject: [PATCH 6/9] Add missing Requires: psycopg2, wtforms --- diff --git a/execdb.spec b/execdb.spec index 250e37c..303446b 100644 --- a/execdb.spec +++ b/execdb.spec @@ -16,6 +16,8 @@ Requires: python3-flask-sqlalchemy Requires: python3-flask-wtf Requires: python3-flask-login Requires: python3-flask-restful +Requires: python3-psycopg2 +Requires: python3-wtforms Requires: python3-six BuildRequires: python3-devel From 5a7fd46cf2f79f17b2260cf4b8accf180ea84b9d Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Feb 04 2019 13:17:41 +0000 Subject: [PATCH 7/9] updating buildbot status info handling code, changed some wording on the index --- diff --git a/execdb/controllers/main.py b/execdb/controllers/main.py index 623e888..cb4f4de 100644 --- a/execdb/controllers/main.py +++ b/execdb/controllers/main.py @@ -32,6 +32,7 @@ from sqlalchemy import desc import json import re +from datetime import datetime from pprint import pformat @@ -173,7 +174,6 @@ def show_steps(uuid): return jsonify(steps) - @main.route('/jobs', methods=['POST']) def create_job(): job = Job() @@ -199,139 +199,78 @@ def create_job(): return jsonify(retval), 201 -def process_event(data): - - def bb_convert_properties(prop): - """Converts list of lists to dict""" - return dict([(key, value) for key, value, _ in prop]) - - # at the moment, we act just on these events - event = data['event'] - known_events = ['changeAdded', 'buildStarted', 'stepStarted', - 'stepFinished', 'buildFinished'] - - if event not in known_events: - # FIXME remove - if 'uuid' in json.dumps(data): - app.logger.debug("UUID found in %s", event) - - return 'Skipping event', 204 - - # grab the 'properties' field - if event == 'changeAdded': - properties = bb_convert_properties(data['payload']['change']['properties']) - elif event in ['buildStarted', 'buildFinished']: - properties = bb_convert_properties(data['payload']['build']['properties']) - elif event in ['stepStarted', 'stepFinished']: - properties = bb_convert_properties(data['payload']['properties']) +def process_bb_status(status_data): + # grab uuid, build state, properties + build_properties = status_data['properties'] + uuid = build_properties['uuid'][0] + build_complete = status_data['complete'] - # abort if uuid is not provided - try: - uuid = properties['uuid'] - except KeyError: - return 'Missing `uuid` field in properties', 400 - - if uuid is None: - return 'UUID set to None', 400 + app.logger.info("Processing data for job {} (complete: {})".format(uuid, build_complete)) + # find job in db try: job = db.session.query(Job).filter(Job.uuid == uuid).one() except orm_exc.NoResultFound: + app.logger.info("UUID {} not found".format(uuid)) return 'UUID not found', 400 - if event == 'changeAdded': - # FIXME ? - pass + # if 'complete' is false, create job + if not build_complete: + app.logger.debug("%s -- adding job %s"% (uuid, status_data['number'])) - elif event == 'buildStarted' and job.current_state == 'Triggered': - job.start() + job.t_build_started = datetime.fromtimestamp(status_data['started_at']) + job.t_triggered = datetime.fromtimestamp(status_data['started_at']) - job.taskname = properties['taskname'] - job.item = properties['item'] - job.item_type = properties['item_type'] - job.arch = properties['arch'] - job.slavename = properties['slavename'] + job.taskname = build_properties['taskname'][0] + job.item = build_properties['item'][0] + job.item_type = build_properties['item_type'][0] + job.arch = build_properties['arch'][0] + job.slavename = build_properties['slavename'][0] job.link_build_log = '/builders/%s/builds/%s' % ( - data['payload']['build']['builderName'], - properties['buildnumber']) + status_data['buildrequest']['builderid'], + status_data['number']) db.session.add(job) - # add 'empty' steps for the build (since we know them already) -# app.logger.debug("%s: %s" % (uuid, data['payload']['build']['steps'])) -# app.logger.debug("%s - Build Started" % uuid) - for step_info in data['payload']['build']['steps']: - # app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) - step = BuildStep(name=step_info['name']) - step.job = job - db.session.add(step) - db.session.commit() - elif event == 'stepStarted' and job.current_state == 'Running': - step_info = data['payload']['step'] -# app.logger.debug("%s - Step Started - %s"% (uuid, step_info['name'])) - try: - step = job.get_build_step(step_info['name']) - except KeyError: - app.logger.debug("Job %s had missing step %s", job.uuid, step_info) - step = BuildStep(name=step_info['name']) - step.job = job - - step.start() - step.status = 'INPROGRESS' - step.data = json.dumps(data['payload']) # FIXME - store sensible subset of data - db.session.add(step) - db.session.commit() -# app.logger.debug("%s - Step Started - %s - written to db"% (uuid, step_info['name'])) + # if 'complete' is true, fill in buildsteps, finish job + else: + # add the completed time and state - elif event == 'stepFinished' and job.current_state == 'Running': - step_info = data['payload']['step'] -# app.logger.debug("%s - Step Finished - %s"% (uuid, step_info['name'])) - try: - step = job.get_build_step(step_info['name']) - except KeyError: - return 'StepFinished received for non-existing step: %r' % step_info['name'], 400 + job.t_build_ended = datetime.fromtimestamp(status_data['complete_at']) + # add the build steps + for step_info in status_data['steps']: - step.finish() + app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) + app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) + step = BuildStep(name=step_info['name']) + step.job = job + step.started_at = datetime.fromtimestamp(step_info['started_at']) + step.finished_at = datetime.fromtimestamp(step_info['complete_at']) + step.data = step_info['state_string'] - step.status = 'OK' - # results key is only present for non-ok results - if 'results' in step_info.keys(): - step.status = 'NOT OK' - step.data = json.dumps(data['payload']) # FIXME - store sensible subset of data + # there doesn't seem to be a really reasonable way to tell if a step has failed but + # this should work well enough for now + if 'failed' in step_info['state_string']: + step.status = 'NOT OK' + else: + step.status = 'OK' - db.session.add(step) - db.session.commit() -# app.logger.debug("%s - Step Finished - %s - written to db" % (uuid, step_info['name'])) + db.session.add(step) - elif event == 'buildFinished' and job.current_state == 'Running': - job.finish() - db.session.add(job) db.session.commit() -# app.logger.debug("%s - Build Finished " % uuid) -@main.route('/buildbottest', methods=['POST']) +@main.route('/buildbot', methods=['POST']) def bb_push(): """ Receives the post-push notifications from buildbot and fills in the steps for the job. """ - # data are embedded in form field 'packets' - data = request.form - try: - data = request.form['packets'] - except werkzeug.exceptions.BadRequestKeyError: - return 'Field `packets` missing in request form.', 400 - data = json.loads(data) - - # app.logger.debug(pformat(data)) - # multiple messages may be present in one 'packet' - for entry in data: - process_event(entry) -# app.logger.debug("%s %s, %s", entry['id'], entry['event'], process_event(entry)) + data = request.get_json() + process_bb_status(data) # plain 200 code needs to be returned - otherwise buildbot is # endlessly trying to re-send the message. diff --git a/execdb/templates/index.html b/execdb/templates/index.html index e01857b..4b86845 100644 --- a/execdb/templates/index.html +++ b/execdb/templates/index.html @@ -8,7 +8,7 @@ Item State Build Steps - Moar + {% for job in jobs -%} @@ -39,7 +39,7 @@ - Detail + Details {% endfor -%} From 026cf866bd7f1a7ceddf3c589d022ed0f4e976da Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Feb 05 2019 12:09:17 +0000 Subject: [PATCH 8/9] removing extra reset of t_triggered --- diff --git a/execdb/controllers/main.py b/execdb/controllers/main.py index cb4f4de..b16d4b5 100644 --- a/execdb/controllers/main.py +++ b/execdb/controllers/main.py @@ -122,7 +122,7 @@ def show_job(uuid): job = db.session.query(Job).filter(Job.uuid == uuid).one() except orm_exc.NoResultFound: return 'UUID not found', 404 - job.t_triggered = str(job.t_triggered).split('.')[0] + return render_template('show_job.html', job=job, buildbot_url=BB_URL, From 383859a558858cd9a087a4609297c5f9edd9fe1f Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Feb 05 2019 12:16:31 +0000 Subject: [PATCH 9/9] removing correct extra reset of t_triggered --- diff --git a/execdb/controllers/main.py b/execdb/controllers/main.py index b16d4b5..6a0aeac 100644 --- a/execdb/controllers/main.py +++ b/execdb/controllers/main.py @@ -123,6 +123,7 @@ def show_job(uuid): except orm_exc.NoResultFound: return 'UUID not found', 404 + job.t_triggered = str(job.t_triggered).split('.')[0] return render_template('show_job.html', job=job, buildbot_url=BB_URL, @@ -219,7 +220,6 @@ def process_bb_status(status_data): app.logger.debug("%s -- adding job %s"% (uuid, status_data['number'])) job.t_build_started = datetime.fromtimestamp(status_data['started_at']) - job.t_triggered = datetime.fromtimestamp(status_data['started_at']) job.taskname = build_properties['taskname'][0] job.item = build_properties['item'][0]