From 4a118781683cb8cb5d02cb33b3939a2a3b80bd76 Mon Sep 17 00:00:00 2001 From: Matt Jia Date: Mar 14 2017 06:02:24 +0000 Subject: [PATCH 1/3] HTTP API for fetching a single waiver --- diff --git a/README.md b/README.md index e6c1501..8fc8b65 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Run the server: $ DEV=true python runapp.py -The server is now running at and aPI calls can be sent to +The server is now running at and API calls can be sent to . All data is stored inside `/var/tmp/waiverdb_db.sqlite`. ## Adjusting configuration diff --git a/tests/test_api_v10.py b/tests/test_api_v10.py index e9bb62b..003990c 100644 --- a/tests/test_api_v10.py +++ b/tests/test_api_v10.py @@ -11,6 +11,7 @@ import pytest import json +from .utils import create_waiver def test_create_waiver(client, session): data = { @@ -38,3 +39,20 @@ def test_create_waiver_with_malformed_data(client): res_data = json.loads(r.data) assert r.status_code == 400 assert 'invalid literal for int()' in res_data['message']['result_id'] + +def test_get_waiver(client, session): + # create a new waiver + waiver = create_waiver(session, result_id=123, username='foo', + product_version='foo-1', comment='bla bla bla') + r = client.get('/api/v1.0/waivers/%s' % waiver.id) + res_data = json.loads(r.data) + assert r.status_code == 200 + assert res_data['username'] == waiver.username + assert res_data['result_id'] == waiver.result_id + assert res_data['product_version'] == waiver.product_version + assert res_data['waived'] == True + assert res_data['comment'] == waiver.comment + +def test_404_for_nonexistent_waiver(client, session): + r = client.get('/api/v1.0/waivers/foo') + assert r.status_code == 404 diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..18d5669 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,20 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +from waiverdb.models import Waiver + +def create_waiver(session, result_id, username, product_version, waived=True, + comment=None): + waiver = Waiver(result_id, username, product_version, waived, comment) + session.add(waiver) + session.flush() + return waiver diff --git a/waiverdb/api_v1.py b/waiverdb/api_v1.py index 95ab07f..88319b8 100644 --- a/waiverdb/api_v1.py +++ b/waiverdb/api_v1.py @@ -11,13 +11,14 @@ # from flask import Blueprint -from flask_restful import reqparse -from werkzeug.exceptions import HTTPException +from flask_restful import Resource, Api, reqparse, marshal_with +from werkzeug.exceptions import HTTPException, BadRequest, NotFound from waiverdb.models import db, Waiver -from waiverdb.utils import to_json +from waiverdb.fields import waiver_fields -api = Blueprint('api_v1', __name__) +api_v1 = (Blueprint('api_v1', __name__)) +api = Api(api_v1) # RP contains request parsers (reqparse.RequestParser). # Parsers are added in each 'resource section' for better readability @@ -29,20 +30,28 @@ RP['create_waiver'].add_argument('waived', type=bool, required=True, location='j RP['create_waiver'].add_argument('product_version', type=str, required=True, location='json') RP['create_waiver'].add_argument('comment', type=str, default=None, location='json') -@api.route('/waivers/', methods=['POST']) -@to_json -def create_waiver(): - try: - args = RP['create_waiver'].parse_args() - except HTTPException as error: - return error.data, error.code - - # hardcode the username for now - username = 'mjia' - waiver = Waiver(args['result_id'], username, args['product_version'], args['waived'], - args['comment']) - - db.session.add(waiver) - db.session.commit() - return waiver, 201 +class WaiversResource(Resource): + @marshal_with(waiver_fields) + def post(self): + args = RP['create_waiver'].parse_args() + # hardcode the username for now + username = 'mjia' + waiver = Waiver(args['result_id'], username, args['product_version'], args['waived'], + args['comment']) + db.session.add(waiver) + db.session.commit() + return waiver, 201 + + +class WaiverResource(Resource): + @marshal_with(waiver_fields) + def get(self, waiver_id): + try: + return Waiver.query.get_or_404(waiver_id) + except: + raise NotFound('Waiver not found') + +# set up the Api resource routing here +api.add_resource(WaiversResource, '/waivers/') +api.add_resource(WaiverResource, '/waivers/') diff --git a/waiverdb/app.py b/waiverdb/app.py index bb436d2..d802344 100644 --- a/waiverdb/app.py +++ b/waiverdb/app.py @@ -13,7 +13,7 @@ import os from flask import Flask from flask_sqlalchemy import SQLAlchemy from waiverdb.logger import init_logging -from waiverdb.api_v1 import api as api_v1 +from waiverdb.api_v1 import api_v1 from waiverdb.models import db def load_default_config(app): diff --git a/waiverdb/config.py b/waiverdb/config.py index 541784e..de70424 100644 --- a/waiverdb/config.py +++ b/waiverdb/config.py @@ -19,6 +19,9 @@ class Config(object): PRODUCTION = False SHOW_DB_URI = False SECRET_KEY = 'replace-me-with-something-random' + # need to explicitly turn this off + # https://github.com/flask-restful/flask-restful/issues/449 + ERROR_404_HELP = False class ProductionConfig(Config): diff --git a/waiverdb/fields.py b/waiverdb/fields.py new file mode 100644 index 0000000..1df599e --- /dev/null +++ b/waiverdb/fields.py @@ -0,0 +1,22 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +from flask_restful import fields + +waiver_fields = { + 'id': fields.Integer, + 'result_id': fields.Integer, + 'username': fields.String, + 'product_version': fields.String, + 'waived': fields.Boolean, + 'comment': fields.String, + 'timestamp': fields.DateTime(dt_format='iso8601'), +} diff --git a/waiverdb/models/waivers.py b/waiverdb/models/waivers.py index f9746ae..1b3533c 100644 --- a/waiverdb/models/waivers.py +++ b/waiverdb/models/waivers.py @@ -32,14 +32,3 @@ class Waiver(db.Model): return '%s(result_id=%r, username=%r, product_version=%r, waived=%r)' % ( self.__class__.__name__, self.result_id, self.username, self.product_version, self.waived) - - def __json__(self): - return { - 'id': self.id, - 'result_id': self.result_id, - 'username': self.username, - 'product_version': self.product_version, - 'waived': self.waived, - 'comment': self.comment, - 'timestamp': self.timestamp.isoformat(), - } diff --git a/waiverdb/utils.py b/waiverdb/utils.py deleted file mode 100644 index c4172d1..0000000 --- a/waiverdb/utils.py +++ /dev/null @@ -1,38 +0,0 @@ - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -import functools -from flask import jsonify - -# https://github.com/miguelgrinberg/api-pycon2015/blob/master/api/decorators.py#L8-L30 -def to_json(f): - """This decorator generates a JSON response from a Python dictionary or - a SQLAlchemy model.""" - @functools.wraps(f) - def wrapped(*args, **kwargs): - rv = f(*args, **kwargs) - status_or_headers = None - headers = None - if isinstance(rv, tuple): - rv, status_or_headers, headers = rv + (None,) * (3 - len(rv)) - if isinstance(status_or_headers, (dict, list)): - headers, status_or_headers = status_or_headers, None - if not isinstance(rv, dict): - # assume it is a model, call its __json__() method - rv = rv.__json__() - - rv = jsonify(rv) - if status_or_headers is not None: - rv.status_code = status_or_headers - if headers is not None: - rv.headers.extend(headers) - return rv - return wrapped From ab49b310ce67bd3f975ca91326a3629032420690 Mon Sep 17 00:00:00 2001 From: Matt Jia Date: Mar 14 2017 06:02:24 +0000 Subject: [PATCH 2/3] HTTP API for fetching a list of waivers --- diff --git a/tests/test_api_v10.py b/tests/test_api_v10.py index 003990c..9cde27a 100644 --- a/tests/test_api_v10.py +++ b/tests/test_api_v10.py @@ -12,6 +12,7 @@ import pytest import json from .utils import create_waiver +import datetime def test_create_waiver(client, session): data = { @@ -56,3 +57,103 @@ def test_get_waiver(client, session): def test_404_for_nonexistent_waiver(client, session): r = client.get('/api/v1.0/waivers/foo') assert r.status_code == 404 + +def test_get_waivers(client, session): + for i in range(0,10): + create_waiver(session, result_id=i, username='foo %d' % i, + product_version='foo-%d' % i, comment='bla bla bla') + r = client.get('/api/v1.0/waivers/') + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 10 + +def test_pagination_waivers(client, session): + for i in range(0,10): + create_waiver(session, result_id=i, username='foo %d' % i, + product_version='foo-%d' % i, comment='bla bla bla') + r = client.get('/api/v1.0/waivers/?page=2&limit=1') + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 1 + assert '/waivers/?limit=1&page=1' in res_data['prev'] + assert '/waivers/?limit=1&page=3' in res_data['next'] + assert '/waivers/?limit=1&page=1' in res_data['first'] + assert '/waivers/?limit=1&page=10' in res_data['last'] + +def test_obsolete_waivers_are_excluded_by_default(client, session): + old_waiver = create_waiver(session, result_id=123, username='foo', + product_version='foo-1') + new_waiver = create_waiver(session, result_id=123, username='foo', + product_version='foo-1', waived=False) + r = client.get('/api/v1.0/waivers/') + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 1 + assert res_data['data'][0]['id'] == new_waiver.id + assert res_data['data'][0]['waived'] == new_waiver.waived + +def test_get_obsolete_waivers(client, session): + old_waiver = create_waiver(session, result_id=123, username='foo', + product_version='foo-1') + new_waiver = create_waiver(session, result_id=123, username='foo', + product_version='foo-1', waived=False) + r = client.get('/api/v1.0/waivers/?include_obsolete=1') + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 2 + assert res_data['data'][0]['id'] == new_waiver.id + assert res_data['data'][1]['id'] == old_waiver.id + +def test_filtering_waivers_by_result_id(client, session): + create_waiver(session, result_id=123, username='foo-1', product_version='foo-1') + create_waiver(session, result_id=234, username='foo-2', product_version='foo-1') + r = client.get('/api/v1.0/waivers/?result_id=123') + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 1 + assert res_data['data'][0]['result_id'] == 123 + +def test_filtering_waivers_by_product_version(client, session): + create_waiver(session, result_id=123, username='foo-1', product_version='release-1') + create_waiver(session, result_id=124, username='foo-1', product_version='release-2') + r = client.get('/api/v1.0/waivers/?product_version=release-1') + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 1 + assert res_data['data'][0]['product_version'] == 'release-1' + +def test_filtering_waivers_by_username(client, session): + create_waiver(session, result_id=123, username='foo', product_version='foo-1') + create_waiver(session, result_id=124, username='bar', product_version='foo-2') + r = client.get('/api/v1.0/waivers/?username=foo') + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 1 + assert res_data['data'][0]['username'] == 'foo' + +def test_filtering_waivers_by_since(client, session): + before1 = (datetime.datetime.utcnow() - datetime.timedelta(seconds=100)).isoformat() + before2 = (datetime.datetime.utcnow() - datetime.timedelta(seconds=99)).isoformat() + after = (datetime.datetime.utcnow() + datetime.timedelta(seconds=100)).isoformat() + create_waiver(session, result_id=123, username='foo', product_version='foo-1') + r = client.get('/api/v1.0/waivers/?since=%s' % before1) + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 1 + assert res_data['data'][0]['result_id'] == 123 + + r = client.get('/api/v1.0/waivers/?since=%s,%s' % (before1, after)) + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 1 + assert res_data['data'][0]['result_id'] == 123 + + r = client.get('/api/v1.0/waivers/?since=%s' % (after)) + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 0 + + r = client.get('/api/v1.0/waivers/?since=%s,%s' % (before1, before2)) + res_data = json.loads(r.data) + assert r.status_code == 200 + assert len(res_data['data']) == 0 diff --git a/waiverdb/api_v1.py b/waiverdb/api_v1.py index 88319b8..512e853 100644 --- a/waiverdb/api_v1.py +++ b/waiverdb/api_v1.py @@ -13,8 +13,10 @@ from flask import Blueprint from flask_restful import Resource, Api, reqparse, marshal_with from werkzeug.exceptions import HTTPException, BadRequest, NotFound +from sqlalchemy.sql.expression import func from waiverdb.models import db, Waiver +from waiverdb.utils import reqparse_since, json_collection from waiverdb.fields import waiver_fields api_v1 = (Blueprint('api_v1', __name__)) @@ -30,8 +32,42 @@ RP['create_waiver'].add_argument('waived', type=bool, required=True, location='j RP['create_waiver'].add_argument('product_version', type=str, required=True, location='json') RP['create_waiver'].add_argument('comment', type=str, default=None, location='json') +RP['get_waivers'] = reqparse.RequestParser() +RP['get_waivers'].add_argument('result_id', location='args') +RP['get_waivers'].add_argument('product_version', location='args') +RP['get_waivers'].add_argument('username', location='args') +RP['get_waivers'].add_argument('include_obsolete', type=bool, default=False, location='args') +# XXX This matches the since query parameter in resultsdb but I think it would +# be good to use two parameters(since and until). +RP['get_waivers'].add_argument('since', location='args') +RP['get_waivers'].add_argument('page', default=1, type=int, location='args') +RP['get_waivers'].add_argument('limit', default=10, type=int, location='args') class WaiversResource(Resource): + + def get(self): + args = RP['get_waivers'].parse_args() + query = Waiver.query.order_by(Waiver.timestamp.desc()) + if args['result_id']: + query = query.filter(Waiver.result_id.in_(args['result_id'].split(','))) + if args['product_version']: + query = query.filter(Waiver.product_version == args['product_version']) + if args['username']: + query = query.filter(Waiver.username == args['username']) + if args['since']: + try: + since_start, since_end = reqparse_since(args['since']) + except: + raise BadRequest("'since' parameter not in ISO8601 format") + if since_start: + query = query.filter(Waiver.timestamp >= since_start) + if since_end: + query = query.filter(Waiver.timestamp <= since_end) + if not args['include_obsolete']: + subquery = db.session.query(func.max(Waiver.id)).group_by(Waiver.result_id) + query = query.filter(Waiver.id.in_(subquery)) + return json_collection(query, args['page'], args['limit']) + @marshal_with(waiver_fields) def post(self): args = RP['create_waiver'].parse_args() diff --git a/waiverdb/utils.py b/waiverdb/utils.py new file mode 100644 index 0000000..6345407 --- /dev/null +++ b/waiverdb/utils.py @@ -0,0 +1,61 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import datetime +from flask import request, url_for +from flask_restful import marshal +from waiverdb.fields import waiver_fields +from werkzeug.exceptions import NotFound + +def reqparse_since(since): + """ + This parses the since(i.e. 2017-02-13T23:37:58.193281, 2017-02-16T23:37:58.193281) + query parameter and returns a tuple. + """ + start = None + end = None + if ',' in since: + start, end = since.split(',') + else: + start = since + if start: + start = datetime.datetime.strptime(start, "%Y-%m-%dT%H:%M:%S.%f") + if end: + end = datetime.datetime.strptime(end, "%Y-%m-%dT%H:%M:%S.%f") + return start, end + +def json_collection(query, page=1, limit=10): + """ + Helper function for Flask request handlers which want to return + a collection of resources as JSON. + """ + try: + p = query.paginate(page, limit) + except NotFound: + return {'data': [], 'prev': None, 'next': None, 'first': None, 'last':None} + pages = {'data': marshal(p.items, waiver_fields)} + query_pairs = request.args.copy() + if query_pairs: + # remove the page number + query_pairs.pop('page', default=None) + if p.has_prev: + pages['prev'] = url_for(request.endpoint, page=p.prev_num, _external=True, + **query_pairs) + else: + pages['prev'] = None + if p.has_next: + pages['next'] = url_for(request.endpoint, page=p.next_num, _external=True, + **query_pairs) + else: + pages['next'] = None + pages['first'] = url_for(request.endpoint, page=1, _external=True, **query_pairs) + pages['last'] = url_for(request.endpoint, page=p.pages, _external=True, **query_pairs) + return pages From 7a87d3afcd41d40f022206a4d7a80c4100b8dacb Mon Sep 17 00:00:00 2001 From: Matt Jia Date: Mar 14 2017 06:02:24 +0000 Subject: [PATCH 3/3] support jsonp --- diff --git a/tests/test_api_v10.py b/tests/test_api_v10.py index 9cde27a..00226dc 100644 --- a/tests/test_api_v10.py +++ b/tests/test_api_v10.py @@ -157,3 +157,9 @@ def test_filtering_waivers_by_since(client, session): res_data = json.loads(r.data) assert r.status_code == 200 assert len(res_data['data']) == 0 + +def test_jsonp(client, session): + waiver = create_waiver(session, result_id=123, username='foo', product_version='foo-1') + r = client.get('/api/v1.0/waivers/%s?callback=jsonpcallback' % waiver.id) + assert r.mimetype == 'application/javascript' + assert 'jsonpcallback' in r.data diff --git a/waiverdb/api_v1.py b/waiverdb/api_v1.py index 512e853..89f23e7 100644 --- a/waiverdb/api_v1.py +++ b/waiverdb/api_v1.py @@ -16,7 +16,7 @@ from werkzeug.exceptions import HTTPException, BadRequest, NotFound from sqlalchemy.sql.expression import func from waiverdb.models import db, Waiver -from waiverdb.utils import reqparse_since, json_collection +from waiverdb.utils import reqparse_since, json_collection, jsonp from waiverdb.fields import waiver_fields api_v1 = (Blueprint('api_v1', __name__)) @@ -44,7 +44,7 @@ RP['get_waivers'].add_argument('page', default=1, type=int, location='args') RP['get_waivers'].add_argument('limit', default=10, type=int, location='args') class WaiversResource(Resource): - + @jsonp def get(self): args = RP['get_waivers'].parse_args() query = Waiver.query.order_by(Waiver.timestamp.desc()) @@ -68,6 +68,7 @@ class WaiversResource(Resource): query = query.filter(Waiver.id.in_(subquery)) return json_collection(query, args['page'], args['limit']) + @jsonp @marshal_with(waiver_fields) def post(self): args = RP['create_waiver'].parse_args() @@ -81,6 +82,7 @@ class WaiversResource(Resource): class WaiverResource(Resource): + @jsonp @marshal_with(waiver_fields) def get(self, waiver_id): try: diff --git a/waiverdb/utils.py b/waiverdb/utils.py index 6345407..741ab81 100644 --- a/waiverdb/utils.py +++ b/waiverdb/utils.py @@ -10,7 +10,8 @@ # GNU General Public License for more details. import datetime -from flask import request, url_for +import functools +from flask import request, url_for, jsonify from flask_restful import marshal from waiverdb.fields import waiver_fields from werkzeug.exceptions import NotFound @@ -59,3 +60,20 @@ def json_collection(query, page=1, limit=10): pages['first'] = url_for(request.endpoint, page=1, _external=True, **query_pairs) pages['last'] = url_for(request.endpoint, page=p.pages, _external=True, **query_pairs) return pages + +def jsonp(func): + """Wraps Jsonified output for JSONP requests.""" + @functools.wraps(func) + def wrapped(*args, **kwargs): + callback = request.args.get('callback', False) + if callback: + resp = jsonify(func(*args, **kwargs)) + resp.set_data('{}({})'.format( + str(callback), + resp.get_data() + )) + resp.mimetype = 'application/javascript' + return resp + else: + return func(*args, **kwargs) + return wrapped