From 0c12a7043accfee184287e4196ae53714dcc2c6e Mon Sep 17 00:00:00 2001
From: Dan Callaghan <dcallagh@redhat.com>
Date: Feb 14 2018 09:51:11 +0000
Subject: database migrations should be a pre-deployment hook


... not a post-start pod hook for the web app. This way, we can see the
logs of the migration pod, and if the migration fails the deployment
will be aborted.

The database migration pod has to first wait for the Postgres pod to
finish starting up, because in a freshly created environment (as in the
tests) the migration pod is started concurrently with the Postgres pod.

Fixes #121.

---

diff --git a/openshift/waiverdb-test-template.yaml b/openshift/waiverdb-test-template.yaml
index c59080e..ff230ae 100644
--- a/openshift/waiverdb-test-template.yaml
+++ b/openshift/waiverdb-test-template.yaml
@@ -170,6 +170,17 @@ objects:
     selector:
       environment: "test-${TEST_ID}"
       service: web
+    strategy:
+      type: Rolling
+      rollingParams:
+        pre:
+          failurePolicy: Abort
+          execNewPod:
+            containerName: web
+            command: [ /bin/sh, -i, -c, "waiverdb wait-for-db && waiverdb db upgrade" ]
+            volumes:
+            - config-volume
+            - secret-volume
     template:
       metadata:
         labels:
@@ -181,10 +192,6 @@ objects:
           image: "docker-registry.engineering.redhat.com/factory2/waiverdb:${WAIVERDB_APP_VERSION}"
           ports:
           - containerPort: 8080
-          lifecycle:
-            postStart:
-              exec:
-                command: [ /bin/sh, -i, -c, "waiverdb db upgrade" ]
           volumeMounts:
           - name: config-volume
             mountPath: /etc/waiverdb
diff --git a/waiverdb/manage.py b/waiverdb/manage.py
index c69b971..1bab403 100644
--- a/waiverdb/manage.py
+++ b/waiverdb/manage.py
@@ -1,7 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0+
 
+import time
 import click
 from flask.cli import FlaskGroup
+from sqlalchemy.exc import OperationalError
+from waiverdb.models import db
 
 
 def create_waiver_app(_):
@@ -14,5 +17,23 @@ def cli():
     pass
 
 
+@cli.command(name='wait-for-db')
+def wait_for_db():
+    """
+    Wait until database server is reachable.
+    """
+    poll_interval = 10  # seconds
+    while True:
+        try:
+            db.engine.connect()
+        except OperationalError as e:
+            click.echo('Failed to connect to database: {}'.format(e))
+            click.echo('Sleeping for {} seconds...'.format(poll_interval))
+            time.sleep(poll_interval)
+            click.echo('Retrying...')
+        else:
+            break
+
+
 if __name__ == '__main__':
     cli()  # pylint: disable=E1120