From 067e9db4c05ba36644fd5aec27400d34cdb516bf Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Jun 14 2017 17:25:09 +0000 Subject: An initial implementation of the database models. This is an initial implementation of a set of database models for policies and rules. It includes an initial Alembic migration for the database and all the boilerplate for setting up the database. It does not currently rely on Flask-SQLAlchemy, but it would be simple to drop it in and it offers some nice additional functionality (pagination, for example). This is mostly just to gather feedback. Signed-off-by: Jeremy Cline --- diff --git a/MANIFEST.in b/MANIFEST.in index 14a820d..51acaa8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include COPYING README.md requirements.txt dev-requirements.txt +include COPYING README.md requirements.txt dev-requirements.txt alembic.ini diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..a03bf80 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,68 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = greenwave:db/migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +sourceless = false + +# version location specification; this defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migrations/versions + +# the output encoding used when revision files +# are written from script.py.mako +output_encoding = utf-8 + +sqlalchemy.url = postgres://postgres:pass@localhost/greenwave + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index 6ed3b86..424ce46 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -12,6 +12,8 @@ import os from flask import Flask + +from . import db from greenwave.logger import init_logging from greenwave.api_v1 import api @@ -43,6 +45,7 @@ def create_app(config_obj=None): raise Warning("You need to change the app.secret_key value for production") # initialize logging init_logging(app) + db.initialize(app.config) # register blueprints app.register_blueprint(api, url_prefix="/api/v1.0") return app diff --git a/greenwave/config.py b/greenwave/config.py index 1dd8ec1..ecba71c 100644 --- a/greenwave/config.py +++ b/greenwave/config.py @@ -21,6 +21,7 @@ class Config(object): SECRET_KEY = 'replace-me-with-something-random' RESULTSDB_API_URL = 'https://taskotron.fedoraproject.org/resultsdb_api/api/v2.0' WAIVERDB_API_URL = 'https://waiverdb.fedoraproject.org/api/v1.0' + DB_URL = 'postgres://postgres:somepassword@localhost/greenwave' REQUESTS_TIMEOUT = (6.1, 15) diff --git a/greenwave/db/__init__.py b/greenwave/db/__init__.py new file mode 100644 index 0000000..0146a7b --- /dev/null +++ b/greenwave/db/__init__.py @@ -0,0 +1,4 @@ +"""This package contains all the database-related modules.""" + +from .meta import initialize, Session, Base # noqa: F401 +from .models import Policy, Rule # noqa: F401 diff --git a/greenwave/db/meta.py b/greenwave/db/meta.py new file mode 100644 index 0000000..63cb35d --- /dev/null +++ b/greenwave/db/meta.py @@ -0,0 +1,62 @@ +""" +This module sets up the basic database objects that all our other modules will +rely on. This includes the declarative base class and global scoped session. +""" +from __future__ import unicode_literals + +from sqlalchemy import create_engine, event +from sqlalchemy.ext import declarative +from sqlalchemy.orm import sessionmaker, scoped_session + + +#: A thread-local session factory. +#: Before you can use this, you must call :func:`initialize`. +Session = scoped_session(sessionmaker()) + + +def initialize(config): + """ + Initialize the database. + + This creates a database engine from the provided configuration and + configures the scoped session to use the engine. + + Args: + config (dict): A dictionary that contains the configuration necessary + to initialize the database. + + Returns: + sqlalchemy.engine: The database engine created from the configuration. + """ + engine = create_engine(config['DB_URL'], echo=config.get('SQL_DEBUG', False)) + if config['DB_URL'].startswith('sqlite:'): + # Flip on foreign key constraints if the database in use is SQLite. See + # http://docs.sqlalchemy.org/en/latest/dialects/sqlite.html#foreign-key-support + event.listen( + engine, + 'connect', + lambda db_con, con_record: db_con.execute('PRAGMA foreign_keys=ON') + ) + Session.configure(bind=engine) + return engine + + +class DeclarativeBaseMixin(object): + """ + A mix-in class for the declarative base class. + + This provides a place to attach functionality that should be available on + all models derived from the declarative base. + + Attributes: + query (sqlalchemy.orm.query.Query): a class property which produces a + Query object against the class and the current Session when called. + Classes that want a customized Query class should sub-class + :class:`sqlalchemy.orm.query.Query` and explicitly set the query + property on the model. + """ + query = Session.query_property() + + +#: The SQLAlchemy declarative base class all models must sub-class. +Base = declarative.declarative_base(cls=DeclarativeBaseMixin) diff --git a/greenwave/db/migrations/README.rst b/greenwave/db/migrations/README.rst new file mode 100644 index 0000000..5ed0ce9 --- /dev/null +++ b/greenwave/db/migrations/README.rst @@ -0,0 +1,13 @@ +This is a generic single-database configuration for Alembic. + +Alembic integrates with SQLAlchemy and it is possible to auto-generate some +migrations:: + + $ alembic revision --autogenerate -m "" + +Note that there are some changes that cannot be detected using auto-generated +Consult the `auto-generate documentation`_ for complete details. + + +.. _auto-generate documentation: + http://alembic.zzzcomputing.com/en/latest/autogenerate.html diff --git a/greenwave/db/migrations/env.py b/greenwave/db/migrations/env.py new file mode 100644 index 0000000..c3b10f9 --- /dev/null +++ b/greenwave/db/migrations/env.py @@ -0,0 +1,72 @@ +from __future__ import with_statement + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from greenwave import db + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here for 'autogenerate' support +target_metadata = db.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/greenwave/db/migrations/script.py.mako b/greenwave/db/migrations/script.py.mako new file mode 100644 index 0000000..6e8de68 --- /dev/null +++ b/greenwave/db/migrations/script.py.mako @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/greenwave/db/migrations/versions/d3e7058cb97e_initial_database.py b/greenwave/db/migrations/versions/d3e7058cb97e_initial_database.py new file mode 100644 index 0000000..ab7b910 --- /dev/null +++ b/greenwave/db/migrations/versions/d3e7058cb97e_initial_database.py @@ -0,0 +1,40 @@ +""" +Initial database + +Revision ID: d3e7058cb97e +Revises: None +Create Date: 2017-06-14 12:42:26.936727 +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd3e7058cb97e' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'policy', + sa.Column('name', sa.Unicode(length=256), nullable=False), + sa.Column('product', sa.Unicode(length=256), nullable=False), + sa.PrimaryKeyConstraint('name', 'product') + ) + op.create_table( + 'rule', + sa.Column('test_case', sa.Unicode(length=256), nullable=False), + sa.Column('policy_name', sa.Unicode(length=256), nullable=False), + sa.Column('policy_product', sa.Unicode(length=256), nullable=False), + sa.ForeignKeyConstraint( + ['policy_name', 'policy_product'], ['policy.name', 'policy.product']), + sa.PrimaryKeyConstraint('test_case', 'policy_name', 'policy_product') + ) + + +def downgrade(): + op.drop_table('rule') + op.drop_table('policy') diff --git a/greenwave/db/models.py b/greenwave/db/models.py new file mode 100644 index 0000000..5e79575 --- /dev/null +++ b/greenwave/db/models.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +"""The Greenwave database models.""" +from __future__ import unicode_literals + +from sqlalchemy import Column, Unicode, ForeignKeyConstraint +from sqlalchemy.orm import relationship + +from .meta import Base + + +class Rule(Base): + """The model for a :class:`Policy` rule. + + Attributes: + test_case (str): A unicode string that maps to a test case. For example, + 'dist.rpmdiff.comparison.virus_scan'. + policy (Policy): A reference to the related :class:`Policy`. + policy_name (str): Part of the foreign key to a :class:`Policy`. + policy_product (str): Part of the foreign key to a :class:`Policy`. + """ + __tablename__ = 'rule' + __table_args__ = ( + ForeignKeyConstraint(['policy_name', 'policy_product'], ['policy.name', 'policy.product']), + ) + + test_case = Column(Unicode(length=256), primary_key=True) + policy_name = Column(Unicode(length=256), nullable=False, primary_key=True) + policy_product = Column(Unicode(length=256), nullable=False, primary_key=True) + + def check(self, test_results, waivers=None): + """Check the given test results and waivers against this rule. + + Args: + test_results (list): A list of test results. + waivers (list): An optional list of waivers to consider. + + Returns: + bool: True if the rule is satisfied by the given test results and waivers. + """ + pass + + +class Policy(Base): + """The model for a policy. + + Attributes: + name (str): A human-readable name for the policy. + product (str): The product this policy applies to. For example, 'rhel-7'. + rules (sqlalchemy.orm.collections.InstrumentedList): A list of :class:`Rule` + objects that make up this policy. + """ + __tablename__ = 'policy' + + name = Column(Unicode(length=256), primary_key=True) + product = Column(Unicode(length=256), primary_key=True) + rules = relationship('Rule', backref='policy') + + def check(self): + """Check this policy to see if it is satisfied.""" + # Maybe here we determine what test of test results we need based on the rules + # in the policy. + test_results = [] + waivers = [] + return [rule.check(test_results, waivers) for rule in self.rules]