From 793d04894ddca93e75a6052e4f4cf4b7421cb0da Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Mar 29 2018 05:34:23 +0000 Subject: [PATCH 1/3] remove systemd journal logging support Now that we are deploying in Openshift there is no systemd journal available anywhere. We can just use stdout all the time. Same as: https://pagure.io/greenwave/c/3ac20f770f3b32653068245b5caf40ad874ad95a --- diff --git a/conf/settings.py.example b/conf/settings.py.example index 3f4da46..b314d25 100644 --- a/conf/settings.py.example +++ b/conf/settings.py.example @@ -3,7 +3,6 @@ SECRET_KEY = 'replace-me-with-something-random' #DATABASE_URI = 'postgresql+psycopg2://dbuser:dbpassword@dbhost:dbport/dbname' DATABASE_URI = 'postgresql+psycopg2:///waiverdb' -JOURNAL_LOGGING = False #SHOW_DB_URI = False HOST= '0.0.0.0' PORT = 5004 diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index e10d45e..3add5f7 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -7,7 +7,7 @@ Quick development setup Install packages required by pip to compile some python packages:: - $ sudo dnf install swig systemd-devel openssl-devel cpp gcc + $ sudo dnf install swig openssl-devel cpp gcc Set up a python virtualenv:: diff --git a/requirements.txt b/requirements.txt index 4419aa1..f0a9c4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ Flask-SQLAlchemy SQLAlchemy gssapi flask-oidc -systemd # packages for the unit tests pytest >= 2.4.2 mock diff --git a/waiverdb.spec b/waiverdb.spec index 07cb403..5f8b5f1 100644 --- a/waiverdb.spec +++ b/waiverdb.spec @@ -21,7 +21,6 @@ BuildRequires: python3-flask-restful BuildRequires: python3-flask-sqlalchemy BuildRequires: python3-psycopg2 BuildRequires: python3-gssapi -BuildRequires: python3-systemd BuildRequires: python3-pytest BuildRequires: python3-mock BuildRequires: python3-flask-oidc @@ -38,7 +37,6 @@ BuildRequires: python-flask-restful BuildRequires: python-flask-sqlalchemy BuildRequires: python-psycopg2 BuildRequires: python-gssapi -BuildRequires: systemd-python BuildRequires: pytest BuildRequires: python-mock BuildRequires: python-flask-oidc @@ -58,7 +56,6 @@ Requires: python3-flask-restful Requires: python3-flask-sqlalchemy Requires: python3-psycopg2 Requires: python3-gssapi -Requires: python3-systemd Requires: python3-mock Requires: python3-flask-oidc Requires: python3-click @@ -72,7 +69,6 @@ Requires: python-flask-restful Requires: python-flask-sqlalchemy Requires: python-psycopg2 Requires: python-gssapi -Requires: systemd-python Requires: python-mock Requires: python-flask-oidc Requires: python-click diff --git a/waiverdb/config.py b/waiverdb/config.py index 318ba57..6eb669b 100644 --- a/waiverdb/config.py +++ b/waiverdb/config.py @@ -13,7 +13,6 @@ class Config(object): """ DEBUG = True DATABASE_URI = 'postgresql+psycopg2:///waiverdb' - JOURNAL_LOGGING = False HOST = '0.0.0.0' PORT = 5004 PRODUCTION = False diff --git a/waiverdb/logger.py b/waiverdb/logger.py index 7051982..3f60f8b 100644 --- a/waiverdb/logger.py +++ b/waiverdb/logger.py @@ -2,7 +2,6 @@ import logging import sys -import systemd.journal def log_to_stdout(app, level=logging.INFO): @@ -15,15 +14,6 @@ def log_to_stdout(app, level=logging.INFO): app.logger.addHandler(stream_handler) -def log_to_journal(app, level=logging.INFO): - journal_handler = systemd.journal.JournalHandler() - journal_handler.setLevel(level) - app.logger.addHandler(journal_handler) - - def init_logging(app): log_level = logging.DEBUG if app.debug else logging.INFO - if app.config['JOURNAL_LOGGING']: - log_to_journal(app, level=log_level) - else: - log_to_stdout(app, level=log_level) + log_to_stdout(app, level=log_level) From 16a023c434ee8606c752e100b0bd9704c08bd5fe Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Mar 29 2018 05:34:23 +0000 Subject: [PATCH 2/3] use a more compact log format --- diff --git a/waiverdb/logger.py b/waiverdb/logger.py index 3f60f8b..47a0a2a 100644 --- a/waiverdb/logger.py +++ b/waiverdb/logger.py @@ -5,8 +5,7 @@ import sys def log_to_stdout(app, level=logging.INFO): - fmt = '[%(filename)s:%(lineno)d] ' if app.debug else '%(module)-12s ' - fmt += '%(asctime)s %(levelname)-7s %(message)s' + fmt = '%(asctime)s [pid %(process)5d] %(name)s %(levelname)s %(message)s' datefmt = '%Y-%m-%d %H:%M:%S' stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setLevel(level) From eb8dd89a1e0c32b22ba46ef2bd824539f1504a6d Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Mar 29 2018 05:34:23 +0000 Subject: [PATCH 3/3] simplify logging stuff There is now always just one handler on the root logger, both within the Flask app and also the manage.py server CLI tool. Its log level will be configured as either DEBUG or INFO according to app.config['DEBUG']. All other sources of logging configuration are removed. --- diff --git a/waiverdb/app.py b/waiverdb/app.py index f563b6b..b7f8405 100644 --- a/waiverdb/app.py +++ b/waiverdb/app.py @@ -77,14 +77,14 @@ def create_app(config_obj=None): populate_db_config(app) if app.config['AUTH_METHOD'] == 'OIDC': app.oidc = OpenIDConnect(app) + # initialize logging + init_logging(app) # initialize db db.init_app(app) # initialize db migrations migrations_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'migrations') Migrate(app, db, directory=migrations_dir) - # initialize logging - init_logging(app) # register blueprints app.register_blueprint(api_v1, url_prefix="/api/v1.0") app.add_url_rule('/healthcheck', view_func=healthcheck) diff --git a/waiverdb/config.py b/waiverdb/config.py index 6eb669b..74adc7d 100644 --- a/waiverdb/config.py +++ b/waiverdb/config.py @@ -13,6 +13,8 @@ class Config(object): """ DEBUG = True DATABASE_URI = 'postgresql+psycopg2:///waiverdb' + # We configure logging explicitly, turn off the Flask-supplied log handler. + LOGGER_HANDLER_POLICY = 'never' HOST = '0.0.0.0' PORT = 5004 PRODUCTION = False diff --git a/waiverdb/logger.py b/waiverdb/logger.py index 47a0a2a..d215084 100644 --- a/waiverdb/logger.py +++ b/waiverdb/logger.py @@ -4,15 +4,29 @@ import logging import sys -def log_to_stdout(app, level=logging.INFO): +def log_to_stdout(level=logging.INFO): fmt = '%(asctime)s [pid %(process)5d] %(name)s %(levelname)s %(message)s' datefmt = '%Y-%m-%d %H:%M:%S' stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setLevel(level) stream_handler.setFormatter(logging.Formatter(fmt=fmt, datefmt=datefmt)) - app.logger.addHandler(stream_handler) + logging.getLogger().addHandler(stream_handler) def init_logging(app): log_level = logging.DEBUG if app.debug else logging.INFO - log_to_stdout(app, level=log_level) + log_to_stdout(level=log_level) + # In general we want to see everything from our own code, + # but not detailed debug messages from third-party libraries. + # Note that the log level on the handler above controls what + # will actually appear on stdout. + logging.getLogger().setLevel(logging.INFO) + logging.getLogger('waiverdb').setLevel(logging.DEBUG) + # The SQLALCHEMY_ECHO setting comes from Flask-SQLAlchemy, which translates + # it to echo=True in the call to create_engine(). But SQLAlchemy itself + # warns not to do that if you are configuring Python logging correctly: + # http://docs.sqlalchemy.org/en/latest/core/engines.html#configuring-logging + # We intercept that setting and do it "properly" instead. + if app.config.get('SQLALCHEMY_ECHO'): + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + del app.config['SQLALCHEMY_ECHO'] diff --git a/waiverdb/migrations/alembic.ini b/waiverdb/migrations/alembic.ini index f8ed480..5f8ee17 100644 --- a/waiverdb/migrations/alembic.ini +++ b/waiverdb/migrations/alembic.ini @@ -7,39 +7,3 @@ # set to 'true' to run the environment during # the 'revision' command, regardless of autogenerate # revision_environment = false - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/waiverdb/migrations/env.py b/waiverdb/migrations/env.py index 5e3e457..55d2ddd 100644 --- a/waiverdb/migrations/env.py +++ b/waiverdb/migrations/env.py @@ -1,16 +1,12 @@ from alembic import context from sqlalchemy import engine_from_config, pool -from logging.config import fileConfig import logging # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) logger = logging.getLogger('alembic.env') # add your model's MetaData object here