From 2854b57a4986bf63a3a14474523650cbb795b62f Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Feb 05 2019 13:12:21 +0000 Subject: Updating buildbot status info handling code to handle new buildbot --- diff --git a/execdb/controllers/main.py b/execdb/controllers/main.py index 623e888..6a0aeac 100644 --- a/execdb/controllers/main.py +++ b/execdb/controllers/main.py @@ -32,6 +32,7 @@ from sqlalchemy import desc import json import re +from datetime import datetime from pprint import pformat @@ -121,6 +122,7 @@ def show_job(uuid): job = db.session.query(Job).filter(Job.uuid == uuid).one() except orm_exc.NoResultFound: return 'UUID not found', 404 + job.t_triggered = str(job.t_triggered).split('.')[0] return render_template('show_job.html', job=job, @@ -173,7 +175,6 @@ def show_steps(uuid): return jsonify(steps) - @main.route('/jobs', methods=['POST']) def create_job(): job = Job() @@ -199,139 +200,77 @@ def create_job(): return jsonify(retval), 201 -def process_event(data): - - def bb_convert_properties(prop): - """Converts list of lists to dict""" - return dict([(key, value) for key, value, _ in prop]) - - # at the moment, we act just on these events - event = data['event'] - known_events = ['changeAdded', 'buildStarted', 'stepStarted', - 'stepFinished', 'buildFinished'] +def process_bb_status(status_data): + # grab uuid, build state, properties + build_properties = status_data['properties'] + uuid = build_properties['uuid'][0] + build_complete = status_data['complete'] - if event not in known_events: - # FIXME remove - if 'uuid' in json.dumps(data): - app.logger.debug("UUID found in %s", event) - - return 'Skipping event', 204 - - # grab the 'properties' field - if event == 'changeAdded': - properties = bb_convert_properties(data['payload']['change']['properties']) - elif event in ['buildStarted', 'buildFinished']: - properties = bb_convert_properties(data['payload']['build']['properties']) - elif event in ['stepStarted', 'stepFinished']: - properties = bb_convert_properties(data['payload']['properties']) - - # abort if uuid is not provided - try: - uuid = properties['uuid'] - except KeyError: - return 'Missing `uuid` field in properties', 400 - - if uuid is None: - return 'UUID set to None', 400 + app.logger.info("Processing data for job {} (complete: {})".format(uuid, build_complete)) + # find job in db try: job = db.session.query(Job).filter(Job.uuid == uuid).one() except orm_exc.NoResultFound: + app.logger.info("UUID {} not found".format(uuid)) return 'UUID not found', 400 - if event == 'changeAdded': - # FIXME ? - pass + # if 'complete' is false, create job + if not build_complete: + app.logger.debug("%s -- adding job %s"% (uuid, status_data['number'])) - elif event == 'buildStarted' and job.current_state == 'Triggered': - job.start() + job.t_build_started = datetime.fromtimestamp(status_data['started_at']) - job.taskname = properties['taskname'] - job.item = properties['item'] - job.item_type = properties['item_type'] - job.arch = properties['arch'] - job.slavename = properties['slavename'] + job.taskname = build_properties['taskname'][0] + job.item = build_properties['item'][0] + job.item_type = build_properties['item_type'][0] + job.arch = build_properties['arch'][0] + job.slavename = build_properties['slavename'][0] job.link_build_log = '/builders/%s/builds/%s' % ( - data['payload']['build']['builderName'], - properties['buildnumber']) + status_data['buildrequest']['builderid'], + status_data['number']) db.session.add(job) - # add 'empty' steps for the build (since we know them already) -# app.logger.debug("%s: %s" % (uuid, data['payload']['build']['steps'])) -# app.logger.debug("%s - Build Started" % uuid) - for step_info in data['payload']['build']['steps']: - # app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) - step = BuildStep(name=step_info['name']) - step.job = job - db.session.add(step) - db.session.commit() - elif event == 'stepStarted' and job.current_state == 'Running': - step_info = data['payload']['step'] -# app.logger.debug("%s - Step Started - %s"% (uuid, step_info['name'])) - try: - step = job.get_build_step(step_info['name']) - except KeyError: - app.logger.debug("Job %s had missing step %s", job.uuid, step_info) - step = BuildStep(name=step_info['name']) - step.job = job - - step.start() - step.status = 'INPROGRESS' - step.data = json.dumps(data['payload']) # FIXME - store sensible subset of data - db.session.add(step) - db.session.commit() -# app.logger.debug("%s - Step Started - %s - written to db"% (uuid, step_info['name'])) + # if 'complete' is true, fill in buildsteps, finish job + else: + # add the completed time and state - elif event == 'stepFinished' and job.current_state == 'Running': - step_info = data['payload']['step'] -# app.logger.debug("%s - Step Finished - %s"% (uuid, step_info['name'])) - try: - step = job.get_build_step(step_info['name']) - except KeyError: - return 'StepFinished received for non-existing step: %r' % step_info['name'], 400 + job.t_build_ended = datetime.fromtimestamp(status_data['complete_at']) + # add the build steps + for step_info in status_data['steps']: - step.finish() + app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) + app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) + step = BuildStep(name=step_info['name']) + step.job = job + step.started_at = datetime.fromtimestamp(step_info['started_at']) + step.finished_at = datetime.fromtimestamp(step_info['complete_at']) + step.data = step_info['state_string'] - step.status = 'OK' - # results key is only present for non-ok results - if 'results' in step_info.keys(): - step.status = 'NOT OK' - step.data = json.dumps(data['payload']) # FIXME - store sensible subset of data + # there doesn't seem to be a really reasonable way to tell if a step has failed but + # this should work well enough for now + if 'failed' in step_info['state_string']: + step.status = 'NOT OK' + else: + step.status = 'OK' - db.session.add(step) - db.session.commit() -# app.logger.debug("%s - Step Finished - %s - written to db" % (uuid, step_info['name'])) + db.session.add(step) - elif event == 'buildFinished' and job.current_state == 'Running': - job.finish() - db.session.add(job) db.session.commit() -# app.logger.debug("%s - Build Finished " % uuid) -@main.route('/buildbottest', methods=['POST']) +@main.route('/buildbot', methods=['POST']) def bb_push(): """ Receives the post-push notifications from buildbot and fills in the steps for the job. """ - # data are embedded in form field 'packets' - data = request.form - try: - data = request.form['packets'] - except werkzeug.exceptions.BadRequestKeyError: - return 'Field `packets` missing in request form.', 400 - data = json.loads(data) - - # app.logger.debug(pformat(data)) - # multiple messages may be present in one 'packet' - for entry in data: - process_event(entry) -# app.logger.debug("%s %s, %s", entry['id'], entry['event'], process_event(entry)) + data = request.get_json() + process_bb_status(data) # plain 200 code needs to be returned - otherwise buildbot is # endlessly trying to re-send the message. diff --git a/execdb/templates/index.html b/execdb/templates/index.html index e01857b..4b86845 100644 --- a/execdb/templates/index.html +++ b/execdb/templates/index.html @@ -8,7 +8,7 @@ Item State Build Steps - Moar + {% for job in jobs -%} @@ -39,7 +39,7 @@ - Detail + Details {% endfor -%}