From a76bdfa464e806300ba2113005f10a04a22751e5 Mon Sep 17 00:00:00 2001
From: Josef Skladanka <jskladan@redhat.com>
Date: Jun 04 2018 15:52:53 +0000
Subject: [PATCH 1/3] Enable grabbing secrets from Vault server


---

diff --git a/data/ansible/runner.yml b/data/ansible/runner.yml
index 69c4cb6..321ae2a 100644
--- a/data/ansible/runner.yml
+++ b/data/ansible/runner.yml
@@ -81,6 +81,12 @@
       register: result
       until: result.rc == 0
 
+    - name: Upload secrets
+      copy:
+        src: '{{taskotron_secrets_file}}'
+        dest: '{{taskotron_secrets_file}}'
+      when: not local
+
     - name: Start heartbeat
       command: >
         ./heartbeat.sh start {{ heartbeat_file }} {{ heartbeat_interval }}
@@ -95,6 +101,15 @@
       static: yes
       # variable 'task' is registered here
 
+    - name: Delete secrets
+      file:
+        state: absent
+        path: '{{taskotron_secrets_file}}'
+      tags:
+        - failsafe
+      when: not local
+
+
     - name: Kill heartbeat
       command: ./heartbeat.sh stop {{ heartbeat_file }}
       when: taskotron_keepalive_minutes|int > 0
diff --git a/libtaskotron/config_defaults.py b/libtaskotron/config_defaults.py
index ebd569c..54c50f9 100644
--- a/libtaskotron/config_defaults.py
+++ b/libtaskotron/config_defaults.py
@@ -74,6 +74,10 @@ class Config(object):
     taskotron_master = 'http://localhost/taskmaster'                        #:
     artifacts_baseurl = 'http://localhost/artifacts'                        #:
     download_cache_enabled = True                                           #:
+    vault_enabled = False                                                   #:
+    vault_server = 'http://localhost:4999/api/v1'                           #:
+    vault_username = 'taskotron'                                            #:
+    vault_password = 'taskotron'                                            #:
 
     tmpdir = '/var/tmp/taskotron'                                           #:
     logdir = '/var/log/taskotron'                                           #:
diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py
index 469f2ad..9fb226c 100644
--- a/libtaskotron/directives/resultsdb_directive.py
+++ b/libtaskotron/directives/resultsdb_directive.py
@@ -109,6 +109,18 @@ summary will be printed out into the log, like this::
 
 directive_class = 'ResultsdbDirective'
 
+def git_origin_url(taskdir):
+    try:
+        gitconfig_filename = os.path.join(taskdir, '.git/config')
+        gitconfig = configparser.ConfigParser()
+        gitconfig.read(gitconfig_filename)
+        task_repo_url = gitconfig['remote "origin"']['url']
+    except TypeError as e:
+        log.exception(e)
+        task_repo_url = None
+    return task_repo_url
+
+
 
 class ResultsdbDirective(BaseDirective):
 
@@ -217,17 +229,11 @@ class ResultsdbDirective(BaseDirective):
         else:
             raise TaskotronDirectiveError('No namespace for task %s exists.' % checkname)
 
-        try:
-            taskdir = os.path.dirname(os.path.abspath(arg_data['task']))
-            gitconfig_filename = os.path.join(taskdir, '.git/config')
-
-            gitconfig = configparser.ConfigParser()
-            gitconfig.read(gitconfig_filename)
-            task_repo_url = gitconfig['remote "origin"']['url']
-        except TypeError as e:
-            log.exception(e)
+        taskdir = os.path.dirname(os.path.abspath(arg_data['task']))
+        task_repo_url = git_origin_url(taskdir)
+        if not task_repo_url:
             raise TaskotronDirectiveError("Could not find task's git remote 'origin' url"
-                                          "in %s" % gitconfig_filename)
+                                          "in %s" % os.path.join(taskdir, '.git/config'))
 
         try:
             if not [ns_repo for ns_repo in ns_repos if task_repo_url.strip().startswith(ns_repo)]:
diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py
index 3d0aa0a..7378914 100644
--- a/libtaskotron/executor.py
+++ b/libtaskotron/executor.py
@@ -12,6 +12,10 @@ import yaml
 import fnmatch
 import signal
 import json
+import tempfile
+import configparser
+import requests
+import re
 
 from libtaskotron import config
 from libtaskotron import image_utils
@@ -181,6 +185,48 @@ class Executor(object):
                 vars_[var] = val
         return vars_
 
+    def _get_vault_secrets(self, taskdir):
+        '''Load secrets from the Vault server and store them in a file
+
+        :param str taskdir: path to the directory with test suite (on overlord)
+        :return: a filename with decrypted secrets
+        '''
+        cfg = config.get_config()
+        secrets = {}
+        if cfg.vault_enabled:
+            task_repo_url = resultsdb_directive.git_origin_url(taskdir)
+            if task_repo_url:
+                re_enabler = re.compile(r'taskotron_enable\(([^)]*)\)')
+                try:
+                    r = requests.get(
+                            "%s/buckets" % cfg.vault_server,
+                            auth=(cfg.vault_username, cfg.vault_password),
+                        )
+                except requests.exceptions.RequestException:
+                    r = None
+                if r and r.ok:
+                    data = r.json()['data']
+                    valid_buckets = []
+                    for b in data:
+                        desc = b['description']
+                        if not desc:
+                            continue
+                        enabled_for = ', '.join(re_enabler.findall(desc))
+                        if not task_repo_url in enabled_for:
+                            continue
+                        valid_buckets.append(b)
+                    for b in valid_buckets:
+                        secrets[b['uuid']] = b['secrets']
+
+        if config.get_config().profile == config.ProfileName.TESTING:
+            return secrets
+
+        fd, fname = tempfile.mkstemp(prefix='taskotron_secrets')
+        os.close(fd)
+        with open(fname, 'w') as fd:
+            fd.write(json.dumps(secrets, indent=2, sort_keys=True))
+        return fname
+
     def _create_playbook_vars(self, test_playbook):
         '''Create and return dictionary containing all variables to be used
         with our ansible playbook.
@@ -230,6 +276,8 @@ class Executor(object):
                 whether VM guest distro has to match taskotron_item
             taskotron_match_host_release
                 whether VM guest release has to match taskotron_item
+            taskotron_secrets_file
+                path to the file with secrets appropriate for the task
             taskotron_supported_arches
                 list of base architectures supported by Taskotron (e.g.
                 'armhfp')
@@ -274,6 +322,7 @@ class Executor(object):
         vars_['taskotron_arch'] = self.arg_data['arch']
         vars_['taskotron_item'] = self.arg_data['item']
         vars_['taskotron_item_type'] = self.arg_data['type']
+        vars_['taskotron_secrets_file'] = self._get_vault_secrets(taskdir=vars_['taskdir'])
         vars_['taskotron_supported_arches'] = cfg.supported_arches
         vars_['taskotron_supported_binary_arches'] = [binarch for arch in
             cfg.supported_arches for binarch in arch_utils.Arches.binary[arch]]
@@ -446,6 +495,7 @@ class Executor(object):
 
         failed = []
         for test_playbook in test_playbooks:
+            playbook_vars = None
             try:
                 # syntax check
                 self._check_playbook_syntax(os.path.join(
@@ -487,7 +537,12 @@ class Executor(object):
                         break
                     else:
                         self.task_vm.teardown()
-
+                try:
+                    if playbook_vars and config.get_config().profile != config.ProfileName.TESTING:
+                        os.remove(playbook_vars['taskotron_secrets_file'])
+                except OSError:
+                    log.info("Could not delete the secrets file at %r",
+                                playbook_vars['taskotron_secrets_file'])
                 log.info('Playbook execution finished: %s', test_playbook)
 
         if failed:
diff --git a/testing/functest_executor.py b/testing/functest_executor.py
index a7c2a48..40771ee 100644
--- a/testing/functest_executor.py
+++ b/testing/functest_executor.py
@@ -89,3 +89,44 @@ class TestExecutor():
         assert len(matched) == 1
         warn_task_line = lines.index(matched[0])
         assert 'skipping' in lines[warn_task_line + 1]
+
+    def test_get_vault_secrets(self, monkeypatch):
+        class MockRequests(object):
+            def __init__(self, retval):
+                self.ok = True
+                self.retval = retval
+                self.exceptions = mock.Mock()
+            def get(self, *args, **kwargs):
+                return self
+            def json(self, *args, **kwargs):
+                return self.retval
+
+        self.conf.vault_enabled = True
+
+        mock_git_origin_url = mock.Mock(return_value='fake://repo.git')
+        monkeypatch.setattr(executor.resultsdb_directive, 'git_origin_url', mock_git_origin_url)
+
+        mock_requests = MockRequests({'data': [
+                {
+                  'description': 'dat description',
+                  'secrets': 'dem secrets',
+                  'uuid': 'dat uuid',
+                }
+             ]})
+        monkeypatch.setattr(executor, 'requests', mock_requests)
+        secrets = self.executor._get_vault_secrets(taskdir=None)
+        assert secrets == {}
+
+
+        mock_requests = MockRequests({'data': [
+                {
+                  'description': 'dat_description, taskotron_enable(fake://repo.git)',
+                  'secrets': 'dem_secrets',
+                  'uuid': 'dat uuid',
+                }
+             ]})
+        monkeypatch.setattr(executor, 'requests', mock_requests)
+        secrets = self.executor._get_vault_secrets(taskdir=None)
+        assert secrets == {'dat uuid': 'dem_secrets'}
+
+

From 8036ea3b3abe38809f9158ee14910bfdc7f9a333 Mon Sep 17 00:00:00 2001
From: Josef Skladanka <jskladan@redhat.com>
Date: Jun 06 2018 13:22:46 +0000
Subject: [PATCH 2/3] Fixed to reflect review


---

diff --git a/conf/taskotron.yaml.example b/conf/taskotron.yaml.example
index 76ab182..13eacc2 100644
--- a/conf/taskotron.yaml.example
+++ b/conf/taskotron.yaml.example
@@ -88,6 +88,19 @@
 ## Please make sure the URL doesn't have a trailing slash.
 #execdb_server: http://localhost:5003
 
+## URL of VAULT server API interface, which stores secrets.
+## Please make sure the URL doesn't have a trailing slash.
+#vault_server: http://localhost:4999/api/v1
+
+## Use vault to grab secrets
+#vault_enabled: False
+
+## Username for vault server
+#vault_username: taskotron
+
+## Password for vault server
+#vault_password: taskotron
+
 ## URL of taskotron buildmaster, to construct log URLs from.
 ## Please make sure the URL doesn't have a trailing slash.
 #taskotron_master: http://localhost/taskmaster
diff --git a/docs/source/writingtasks.rst b/docs/source/writingtasks.rst
index f6cdaae..6b998d4 100644
--- a/docs/source/writingtasks.rst
+++ b/docs/source/writingtasks.rst
@@ -151,6 +151,14 @@ Taskotron-specific variables:
     tasks might want to ignore this (for example, if they're capable of
     checking all architectures in a single run).
 
+``taskotron_secrets_file``
+    (string) Path to the file containing relevant secrets grabbed from the
+    Vault server. If a bucket owned by the taskotron user has a string like
+    ``taskotron_enable(XYZ)`` in the ``description`` field, and the ``XYZ``
+    is the same as the current task's git source repo, the bucket's secrets
+    are included in the file. This file contains a json-encoded dictionary,
+    where bucket's uuids are keys, and a dict of secrets is the respective value.
+
 ``taskotron_supported_arches``
     (list of strings) A list of all base architectures (i.e. ``x86_64``,
     ``i386``, ``armhfp``) that are officially supported by this Taskotron
diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py
index 7378914..cc218bc 100644
--- a/libtaskotron/executor.py
+++ b/libtaskotron/executor.py
@@ -61,6 +61,7 @@ class Executor(object):
         'taskotron_arch',
         'taskotron_item_type',
         'taskotron_item',
+        'taskotron_secrets_file',
         'taskotron_supported_arches',
         'taskotron_supported_binary_arches',
     ]
@@ -196,17 +197,20 @@ class Executor(object):
         if cfg.vault_enabled:
             task_repo_url = resultsdb_directive.git_origin_url(taskdir)
             if task_repo_url:
-                re_enabler = re.compile(r'taskotron_enable\(([^)]*)\)')
                 try:
-                    r = requests.get(
+                    session = file_utils._get_session()
+                    r = session.get(
                             "%s/buckets" % cfg.vault_server,
                             auth=(cfg.vault_username, cfg.vault_password),
                         )
-                except requests.exceptions.RequestException:
+                except requests.exceptions.RequestException as e:
+                    log.error("Connection to Vault server failed. %s", e)
                     r = None
+
                 if r and r.ok:
                     data = r.json()['data']
                     valid_buckets = []
+                    re_enabler = re.compile(r'taskotron_enable\((.*?)\)')
                     for b in data:
                         desc = b['description']
                         if not desc:
@@ -217,6 +221,8 @@ class Executor(object):
                         valid_buckets.append(b)
                     for b in valid_buckets:
                         secrets[b['uuid']] = b['secrets']
+                elif r and not r.ok:
+                    log.error("Could not get data from vault. %r, %r", r.status_code, r.reason)
 
         if config.get_config().profile == config.ProfileName.TESTING:
             return secrets
@@ -540,9 +546,9 @@ class Executor(object):
                 try:
                     if playbook_vars and config.get_config().profile != config.ProfileName.TESTING:
                         os.remove(playbook_vars['taskotron_secrets_file'])
-                except OSError:
-                    log.info("Could not delete the secrets file at %r",
-                                playbook_vars['taskotron_secrets_file'])
+                except OSError as e:
+                    log.warning("Could not delete the secrets file at %r. %s",
+                                playbook_vars['taskotron_secrets_file'], e)
                 log.info('Playbook execution finished: %s', test_playbook)
 
         if failed:
diff --git a/testing/functest_executor.py b/testing/functest_executor.py
index 40771ee..5920f28 100644
--- a/testing/functest_executor.py
+++ b/testing/functest_executor.py
@@ -113,7 +113,8 @@ class TestExecutor():
                   'uuid': 'dat uuid',
                 }
              ]})
-        monkeypatch.setattr(executor, 'requests', mock_requests)
+        mock_session = mock.Mock(return_value=mock_requests)
+        monkeypatch.setattr(executor.file_utils, '_get_session', mock_session)
         secrets = self.executor._get_vault_secrets(taskdir=None)
         assert secrets == {}
 
@@ -125,7 +126,8 @@ class TestExecutor():
                   'uuid': 'dat uuid',
                 }
              ]})
-        monkeypatch.setattr(executor, 'requests', mock_requests)
+        mock_session = mock.Mock(return_value=mock_requests)
+        monkeypatch.setattr(executor.file_utils, '_get_session', mock_session)
         secrets = self.executor._get_vault_secrets(taskdir=None)
         assert secrets == {'dat uuid': 'dem_secrets'}
 

From 8df72216d0654d2236cdc2587a0f578e099ec74b Mon Sep 17 00:00:00 2001
From: Josef Skladanka <jskladan@redhat.com>
Date: Jun 06 2018 13:25:29 +0000
Subject: [PATCH 3/3] .


---

diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py
index cc218bc..f90978f 100644
--- a/libtaskotron/executor.py
+++ b/libtaskotron/executor.py
@@ -535,6 +535,12 @@ class Executor(object):
                     test_playbook, e)
                 failed.append(test_playbook)
             finally:
+                try:
+                    if playbook_vars and config.get_config().profile != config.ProfileName.TESTING:
+                        os.remove(playbook_vars['taskotron_secrets_file'])
+                except OSError as e:
+                    log.warning("Could not delete the secrets file at %r. %s",
+                                playbook_vars['taskotron_secrets_file'], e)
                 if self.task_vm is not None:
                     if self.arg_data['no_destroy']:
                         log.info('Not destroying disposable client as '
@@ -543,12 +549,6 @@ class Executor(object):
                         break
                     else:
                         self.task_vm.teardown()
-                try:
-                    if playbook_vars and config.get_config().profile != config.ProfileName.TESTING:
-                        os.remove(playbook_vars['taskotron_secrets_file'])
-                except OSError as e:
-                    log.warning("Could not delete the secrets file at %r. %s",
-                                playbook_vars['taskotron_secrets_file'], e)
                 log.info('Playbook execution finished: %s', test_playbook)
 
         if failed:

