From 866f7b6dc12379fdb9d8794b98f8a5af9df87221 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 1/17] Add python-six Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/containers/Dockerfile-rpm b/tests/containers/Dockerfile-rpm index 70547d7..0ea3c69 100644 --- a/tests/containers/Dockerfile-rpm +++ b/tests/containers/Dockerfile-rpm @@ -10,4 +10,4 @@ RUN yum install -y make python2-pylint python-pep8 python-openid python-openid-t libsss_simpleifp openldap-servers mod_auth_gssapi krb5-server \ socket_wrapper nss_wrapper python-requests-kerberos python-lesscpy\ nodejs-less krb5-workstation python-sssdconfig sqlite python-jwcrypto \ - mod_ssl mod_auth_openidc python-jwcrypto + mod_ssl mod_auth_openidc python-jwcrypto python-six From 699b9bc548b8828f5bc620f854ac56d808b08099 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 2/17] Add python3 dependencies on Fedora Signed-off-by: Patrick Uiterwijk --- diff --git a/Makefile b/Makefile index 0b54d37..f702692 100644 --- a/Makefile +++ b/Makefile @@ -139,21 +139,22 @@ quickrun: container-quickrun # Testing within containers container-centos6: @echo "Building CentOS 6 container ..." - @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-centos tests/containers/Dockerfile-rpm; echo "USER testuser") | sed -e 's/BASE/centos:6/' | docker build -f - -q -t ipsilon-centos6 - && echo "CentOS 6 container built" || echo "CentOS 6 container build failed (optional)" + @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-centos tests/containers/Dockerfile-rpm tests/containers/Dockerfile-rpm-py2; echo "USER testuser") | sed -e 's/BASE/centos:6/' | docker build -f - -q -t ipsilon-centos6 - && echo "CentOS 6 container built" || echo "CentOS 6 container build failed (optional)" container-centos7: @echo "Building CentOS 7 container ..." - @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-centos tests/containers/Dockerfile-rpm; echo "USER testuser") | sed -e 's/BASE/centos:7/' | docker build -f - -q -t ipsilon-centos7 - + @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-centos tests/containers/Dockerfile-rpm tests/containers/Dockerfile-rpm-py2; echo "USER testuser") | sed -e 's/BASE/centos:7/' | docker build -f - -q -t ipsilon-centos7 - @echo "CentOS 7 container built" container-fedora28: @echo "Building Fedora 28 container ..." - @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-fedora tests/containers/Dockerfile-rpm; echo "USER testuser") | sed -e 's/BASE/fedora:28/' | docker build -f - -q -t ipsilon-fedora28 - + # Fedora 28 is missing python3-lasso. When this gets bumped, use py3 + @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-fedora tests/containers/Dockerfile-rpm tests/containers/Dockerfile-rpm-py2; echo "USER testuser") | sed -e 's/BASE/fedora:28/' | docker build -f - -q -t ipsilon-fedora28 - @echo "Fedora 28 container built" container-fedora29: @echo "Building Fedora 29 container ..." - @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-fedora tests/containers/Dockerfile-rpm; echo "USER testuser") | sed -e 's/BASE/fedora:29/' | docker build -f - -q -t ipsilon-fedora29 - + @(cat tests/containers/Dockerfile-base tests/containers/Dockerfile-fedora tests/containers/Dockerfile-rpm tests/containers/Dockerfile-rpm-py3; echo "USER testuser") | sed -e 's/BASE/fedora:29/' | docker build -f - -q -t ipsilon-fedora29 - @echo "Fedora 29 container built" containers: container-centos6 container-centos7 container-fedora28 container-fedora29 diff --git a/tests/containers/Dockerfile-fedora b/tests/containers/Dockerfile-fedora index 599a4f3..6d3e3ff 100644 --- a/tests/containers/Dockerfile-fedora +++ b/tests/containers/Dockerfile-fedora @@ -1 +1 @@ -RUN yum install -y etcd python2-python-etcd dbus-python python2-ipalib bandit +RUN yum install -y etcd python3-python-etcd dbus-python python3-ipalib bandit diff --git a/tests/containers/Dockerfile-rpm b/tests/containers/Dockerfile-rpm index 0ea3c69..abda712 100644 --- a/tests/containers/Dockerfile-rpm +++ b/tests/containers/Dockerfile-rpm @@ -1,13 +1,8 @@ RUN yum update -y \ && yum install -y which # This should be kept in sync with the develop page on the website. -# Distro-specific packages should go in the distro sub-dockerfiles. -RUN yum install -y make python2-pylint python-pep8 python-openid python-openid-teams \ - python-openid-cla python-cherrypy m2crypto lasso-python \ - python-sqlalchemy python-ldap python-pam python-fedora \ - freeipa-python httpd mod_auth_mellon postgresql-server \ - openssl mod_wsgi python-jinja2 python-psycopg2 sssd \ - libsss_simpleifp openldap-servers mod_auth_gssapi krb5-server \ - socket_wrapper nss_wrapper python-requests-kerberos python-lesscpy\ - nodejs-less krb5-workstation python-sssdconfig sqlite python-jwcrypto \ - mod_ssl mod_auth_openidc python-jwcrypto python-six +# Distro-specific and python packages should go in the distro sub-dockerfiles. +RUN yum install -y make httpd mod_auth_mellon postgresql-server \ + openssl sssd libsss_simpleifp openldap-servers mod_auth_gssapi \ + krb5-server socket_wrapper nss_wrapper nodejs-less krb5-workstation \ + sqlite mod_ssl mod_auth_openidc diff --git a/tests/containers/Dockerfile-rpm-py2 b/tests/containers/Dockerfile-rpm-py2 new file mode 100644 index 0000000..8acc0a5 --- /dev/null +++ b/tests/containers/Dockerfile-rpm-py2 @@ -0,0 +1,6 @@ +RUN yum install -y python2-pylint python-pep8 python-openid python-openid-teams \ + python-openid-cla python-cherrypy m2crypto lasso-python \ + python-sqlalchemy python-ldap python-pam python-fedora \ + mod_wsgi python-jinja2 python-psycopg2 \ + python-requests-kerberos python-lesscpy python-sssdconfig \ + python-jwcrypto python-jwcrypto python-six diff --git a/tests/containers/Dockerfile-rpm-py3 b/tests/containers/Dockerfile-rpm-py3 new file mode 100644 index 0000000..3b0e42d --- /dev/null +++ b/tests/containers/Dockerfile-rpm-py3 @@ -0,0 +1,7 @@ +RUN yum install -y python3-pylint python3-pep8 python3-openid python3-openid-teams \ + python3-openid-cla python3-cherrypy python3-m2crypto python3-lasso \ + python3-sqlalchemy python3-ldap python3-pam python3-fedora \ + python3-ipalib python3-mod_wsgi python3-jinja2 python3-psycopg2 \ + python3-requests-kerberos python3-lesscpy python3-sssdconfig \ + python3-jwcrypto python3-jwcrypto python3-six +RUN yum erase -y python2 From 4d33d23fb52f04270a3dc0140ae8429453c306b1 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 3/17] Use pylint-3 for lint Signed-off-by: Patrick Uiterwijk --- diff --git a/Makefile b/Makefile index f702692..d054d1c 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ all: testdeps lint pep8 test security testdeps: # Determine if test deps are installed # First, some binaries - which pylint-2 + which pylint-3 which pep8 which httpd which postgres @@ -42,7 +42,7 @@ lint: # W0613 - unused argument # Ignore cherrypy class members as they are dynamically added # Ignore IPA API class members as they are dynamically added - pylint-2 -d c,r,i,W0613 -r n -f colorized \ + pylint-3 -d c,r,i,W0613 -r n -f colorized \ --notes= \ --ignored-classes=cherrypy,API \ --disable=star-args \ @@ -78,7 +78,7 @@ cscope: git ls-files | xargs pycscope lp-test: - pylint-2 -d c,r,i,W0613 -r n -f colorized \ + pylint-3 -d c,r,i,W0613 -r n -f colorized \ --notes= \ --ignored-classes=cherrypy \ --disable=star-args \ From 641320871c00b89dbdad878a0499b0054ea214f9 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 4/17] Use logging.warning instead of logging.warn Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/install/ipsilon-client-install b/ipsilon/install/ipsilon-client-install index 101c3cb..f70f72c 100755 --- a/ipsilon/install/ipsilon-client-install +++ b/ipsilon/install/ipsilon-client-install @@ -327,7 +327,7 @@ def saml2_add_arguments(parser): def saml2_verify_arguments(args): if args['saml_auth']: - logger.warn('--saml-auth is deprecated. Please use --auth-location') + logger.warning('--saml-auth is deprecated. Please use --auth-location') args['auth_location'] = args['saml_auth'] # Validate that all path options begin with '/' diff --git a/ipsilon/login/authldap.py b/ipsilon/login/authldap.py index 6e9afd3..efdca38 100644 --- a/ipsilon/login/authldap.py +++ b/ipsilon/login/authldap.py @@ -261,11 +261,11 @@ class Installer(LoginManagerInstaller): lh.search_s(test_dn, ldap.SCOPE_BASE, attrlist=['objectclasses']) except ldap.INSUFFICIENT_ACCESS: - logging.warn('Anonymous access not allowed, continuing') + logging.warning('Anonymous access not allowed, continuing') except ldap.UNWILLING_TO_PERFORM: # probably minSSF issue - logging.warn('LDAP server unwilling to perform, expect issues') + logging.warning('LDAP server unwilling to perform, expect issues') except ldap.SERVER_DOWN: - logging.warn('LDAP server is down') + logging.warning('LDAP server is down') except ldap.NO_SUCH_OBJECT: logging.error('Base DN not found') return False From b5aadbb2e56fe1fa5799813fa0fad75a16c38bfe Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 5/17] getargvaluese is not actually deprecated, docs are just wrong Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/util/log.py b/ipsilon/util/log.py index dc4e242..165274d 100644 --- a/ipsilon/util/log.py +++ b/ipsilon/util/log.py @@ -285,6 +285,9 @@ class Log(object): the instance parameter of a method self, then you could do this. ''' + # Per the python3 docs on getargvalues(): + # "This function was inadvertently marked as deprecated in Python 3.5." + # pylint: disable=deprecated-method args, _, _, value_dict = inspect.getargvalues(frame_obj) # Is the functions first parameter named 'self'? if len(args) and args[0] == 'self': From 4e210a052033c5b70401ec3d6ea99784988b6a31 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 6/17] Add --python-3 flag for test controller Signed-off-by: Patrick Uiterwijk --- diff --git a/runtests b/runtests index e6e16c8..c800eab 100755 --- a/runtests +++ b/runtests @@ -1,3 +1,31 @@ #!/bin/sh +pyver="" +if [ "$1" == "python2" -o "$1" == "python3" ]; +then + pyver="$1" + shift +fi + +if [ "$pyver" == "" ]; +then + echo "Determining Python version" + if which python >/dev/null 2>&1; + then + echo "Using Python" + pyver="python" + elif which python2 >/dev/null 2>&1; + then + echo "Using Python2" + pyver="python2" + elif which python3 >/dev/null 2>&1; + then + echo "Using Python3" + pyver="python3" + else + echo "Unable to find a usable python interpreter" + exit 1 + fi +fi + export PYTHONPATH=. -exec python ./tests/tests.py "$@" +exec $pyver ./tests/tests.py "$@" diff --git a/tests/helpers/common.py b/tests/helpers/common.py index e366db2..3adce6b 100755 --- a/tests/helpers/common.py +++ b/tests/helpers/common.py @@ -78,6 +78,7 @@ class IpsilonTestBase(object): def __init__(self, name, execname, allow_wrappers=True): self.name = name self.execname = execname + self.py3 = False self.rootdir = os.getcwd() self.testdir = None self.testuser = pwd.getpwuid(os.getuid())[0] @@ -88,6 +89,9 @@ class IpsilonTestBase(object): self.stdout = None self.stderr = None + def set_py3(self, use_py3): + self.py3 = use_py3 + def platform_supported(self): """This return whether the current platform supports this test. @@ -197,7 +201,8 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, 'HTTPPORT': port, 'NAME': name, 'CERTROOT': os.path.join(self.testdir, - 'certs')}) + 'certs'), + 'PYTHON3': '_python3' if self.py3 else ''}) filename = os.path.join(httpdir, 'httpd.conf') with open(filename, 'w+') as f: f.write(text) @@ -462,7 +467,7 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, exe = self.execname if exe.endswith('c'): exe = exe[:-1] - return self.run_and_collect([exe], env) + return self.run_and_collect([self.pycmd, exe], env) def run_and_collect(self, cmd, env): p = subprocess.Popen(cmd, env=env, diff --git a/tests/httpd.conf b/tests/httpd.conf index cb766c3..c93452a 100644 --- a/tests/httpd.conf +++ b/tests/httpd.conf @@ -19,7 +19,7 @@ LoadModule ssl_module modules/mod_ssl.so LoadModule alias_module modules/mod_alias.so LoadModule rewrite_module modules/mod_rewrite.so LoadModule version_module modules/mod_version.so -LoadModule wsgi_module modules/mod_wsgi.so +LoadModule wsgi_module modules/mod_wsgi${PYTHON3}.so = 2.4> # openidc needs to be before mellon: https://bugzilla.redhat.com/show_bug.cgi?id=1332729 LoadModule auth_openidc_module modules/mod_auth_openidc.so diff --git a/tests/tests.py b/tests/tests.py index e5a688f..a9cc4e8 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -11,6 +11,7 @@ import argparse from ipsilon.util import plugin import os import sys +import six import subprocess from helpers.common import WRAP_HOSTNAME # pylint: disable=relative-import from helpers.control import TC # pylint: disable=relative-import @@ -43,6 +44,8 @@ def parse_args(): help='Test results header') parser.add_argument('--path', default='%s/testdir' % os.getcwd(), help="Directory in which tests are run") + parser.add_argument('--python-3', '-3', action='store_true', + help='Use python 3') parser.add_argument('--fail-on-first-error', '-x', action='store_true', help='Abort test run on first test failure') parser.add_argument('--test', action='append', default=None, @@ -98,6 +101,8 @@ def try_wrappers(base, wrappers, allow_wrappers): def run_test(testname, test, args): + if args['python_3']: + test.set_py3(True) supported = test.platform_supported() if supported is not None: return (TEST_RESULT_SKIP, supported) @@ -158,6 +163,10 @@ def result_is_fail(result): def main(): args = parse_args() + if six.PY3 and not args['python_3']: + print("Forcing Python3 tests") + args['python_3'] = True + tests = get_tests() if args['list_tests']: for testname in tests.keys(): From f5a80fb21a88a495ac1234897d0e75068444964b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 7/17] Make sure that REST pages encode utf-8 Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/util/endpoint.py b/ipsilon/util/endpoint.py index 49494fc..95d80d9 100644 --- a/ipsilon/util/endpoint.py +++ b/ipsilon/util/endpoint.py @@ -84,13 +84,13 @@ class Endpoint(Log): self.debug("Wrong referer %s in request to %s" % (referer, url)) raise cherrypy.HTTPError(403) - return op(*args, **kwargs) + return op(*args, **kwargs).encode('utf-8') else: op = getattr(self, 'root', None) if callable(op): - return op(*args, **kwargs) + return op(*args, **kwargs).encode('utf-8') - return self.default(*args, **kwargs) + return self.default(*args, **kwargs).encode('utf-8') def default(self, *args, **kwargs): raise cherrypy.NotFound() diff --git a/ipsilon/util/errors.py b/ipsilon/util/errors.py index 7339253..be8c845 100644 --- a/ipsilon/util/errors.py +++ b/ipsilon/util/errors.py @@ -10,7 +10,7 @@ class Errors(Page): output_page = self._template(*args, **kwargs) # for some reason cherrypy will choke if the output # is a unicode object, so use str() here to please it - return str(output_page) + return output_page.encode('utf-8') def handler(self, status, message, traceback, version): self.debug(repr([status, message, traceback, version])) From 9db45b19e164e6ac64630186ff4a2d3021ece51b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 8/17] fconf: store the timestamp of loaded configuration Turns out that we never stored this. It just was never a problem with python2, because comparing a float with None is totally fine there. Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index 70364e6..1741395 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -296,6 +296,7 @@ class FileStore(BaseStore): self._config = ConfigParser.RawConfigParser() self._config.optionxform = str self._config.read(self._filename) + self._timestamp = timestamp return self._config def add_constraint(self, constraint): From 1140fd13b1d88a24ce4f68c0de5e5fce2d767b69 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 9/17] Make core code and test suite Python3 compatible Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/admin/common.py b/ipsilon/admin/common.py index 8c177de..5d82dcd 100644 --- a/ipsilon/admin/common.py +++ b/ipsilon/admin/common.py @@ -2,6 +2,7 @@ import cherrypy import logging +import six from ipsilon.util.page import Page from ipsilon.util.page import admin_protect from ipsilon.util.endpoint import allow_iframe @@ -66,7 +67,7 @@ class AdminPluginConfig(AdminPage): conf = self._po.get_config_obj() - for name, option in conf.iteritems(): + for name, option in six.iteritems(conf): if name in kwargs: value = kwargs[name] if isinstance(option, pconfig.List): @@ -338,7 +339,7 @@ class Admin(AdminPage): def get_complex_list_value(name, old_value, **kwargs): delete = list() change = dict() - for key, val in kwargs.iteritems(): + for key, val in six.iteritems(kwargs): if not key.startswith(name): continue n = key[len(name):] @@ -364,7 +365,7 @@ def get_complex_list_value(name, old_value, **kwargs): del change[i] # perform requested changes - for index, val in change.iteritems(): + for index, val in six.iteritems(change): if val.startswith(('http://', 'https://')): val_list = [val] else: @@ -384,7 +385,7 @@ def get_complex_list_value(name, old_value, **kwargs): # the previous loop may add 'None' entries # if any still exists mark them to be deleted - for i in xrange(0, len(value)): + for i in six.moves.range(0, len(value)): if value[i] is None: delete.append(i) @@ -404,7 +405,7 @@ def get_complex_list_value(name, old_value, **kwargs): def get_mapping_list_value(name, old_value, **kwargs): delete = list() change = dict() - for key, val in kwargs.iteritems(): + for key, val in six.iteritems(kwargs): if not key.startswith(name): continue n = key[len(name):] @@ -433,7 +434,7 @@ def get_mapping_list_value(name, old_value, **kwargs): del change[i] # perform requested changes - for index, fields in change.iteritems(): + for index, fields in six.iteritems(change): for k in 'from', 'to': if k in fields: val = fields[k] @@ -468,7 +469,7 @@ def get_mapping_list_value(name, old_value, **kwargs): # the previous loop may add 'None' entries # if any still exists mark them to be deleted - for i in xrange(0, len(value)): + for i in six.moves.range(0, len(value)): if value[i] is None: delete.append(i) diff --git a/ipsilon/info/infoldap.py b/ipsilon/info/infoldap.py index c4b4888..203eaf6 100644 --- a/ipsilon/info/infoldap.py +++ b/ipsilon/info/infoldap.py @@ -6,6 +6,7 @@ from ipsilon.util.plugin import PluginObject from ipsilon.util.policy import Policy from ipsilon.util import config as pconfig import ldap +import six import subprocess @@ -120,7 +121,7 @@ Info plugin that uses LDAP to retrieve user data. """ elif len(result) > 1: raise Exception('No unique user object could be found!') data = dict() - for name, value in result[0][1].iteritems(): + for name, value in six.iteritems(result[0][1]): if isinstance(value, list) and len(value) == 1: value = value[0] data[name] = value diff --git a/ipsilon/info/infonss.py b/ipsilon/info/infonss.py index 1585e97..3fc66cd 100644 --- a/ipsilon/info/infonss.py +++ b/ipsilon/info/infonss.py @@ -32,11 +32,10 @@ Info plugin that uses the system NSS functions to retrieve user data.""" def _get_posix_groups(self, user, group): groups = set() - getgrouplist = getattr(os, 'getgrouplist', None) - if getgrouplist: + if hasattr(os, 'getgrouplist'): # On python2, None is not callable. On python3, this is a function # pylint: disable=not-callable - ids = getgrouplist(user, group) + ids = os.getgrouplist(user, group) for i in ids: try: g = grp.getgrgid(i) diff --git a/ipsilon/install/ipsilon-client-install b/ipsilon/install/ipsilon-client-install index f70f72c..cead0eb 100755 --- a/ipsilon/install/ipsilon-client-install +++ b/ipsilon/install/ipsilon-client-install @@ -7,9 +7,7 @@ from ipsilon.tools.saml2metadata import SAML2_NAMEID_MAP from ipsilon.tools.saml2metadata import SAML2_SERVICE_MAP from ipsilon.tools.certs import Certificate from ipsilon.tools import files -from urllib import urlencode import argparse -import ConfigParser import getpass import json import logging @@ -17,6 +15,8 @@ import os import pwd import requests import shutil +from six.moves import configparser +from six.moves.urllib.parse import urlencode import socket import sys import base64 @@ -77,7 +77,7 @@ def saml2(): path = os.path.join(HTTPDIR % 'saml2', args['hostname']) if os.path.exists(path): raise Exception('Service Provider is already configured') - os.makedirs(path, 0750) + os.makedirs(path, 0o750) else: path = os.getcwd() @@ -169,7 +169,9 @@ def saml2(): if not args['saml_no_httpd']: idp_metafile = os.path.join(path, 'idp-metadata.xml') - with open(idp_metafile, 'w+') as f: + with open(idp_metafile, 'w+b') as f: + if not isinstance(idpmeta, bytes): + idpmeta = idpmeta.encode('utf-8') f.write(idpmeta) saml_protect = 'auth' @@ -471,7 +473,7 @@ def log_exception(e): def parse_config_profile(args): - config = ConfigParser.ConfigParser() + config = configparser.RawConfigParser() files = config.read(args['config_profile']) if len(files) == 0: raise ConfigurationError('Config Profile file %s not found!' % @@ -590,6 +592,7 @@ if __name__ == '__main__': elif service_type == 'openidc': openidc() except Exception as e: # pylint: disable=broad-except + raise log_exception(e) if 'uninstall' in args and args['uninstall'] is True: logging.info('Uninstallation aborted.') diff --git a/ipsilon/install/ipsilon-db2conf b/ipsilon/install/ipsilon-db2conf index ed93282..9e76889 100755 --- a/ipsilon/install/ipsilon-db2conf +++ b/ipsilon/install/ipsilon-db2conf @@ -7,9 +7,9 @@ import pkg_resources # pylint: disable=unused-import import argparse import cherrypy -from ConfigParser import RawConfigParser import logging import os +from six.moves.configparser import RawConfigParser import sys import unicodedata diff --git a/ipsilon/install/ipsilon-server-install b/ipsilon/install/ipsilon-server-install index 1a4efcb..c43a7c3 100755 --- a/ipsilon/install/ipsilon-server-install +++ b/ipsilon/install/ipsilon-server-install @@ -11,7 +11,6 @@ from ipsilon.helpers.common import EnvHelpersInstall from ipsilon.authz.common import AuthzProviderInstall from ipsilon.util.data import UserStore from ipsilon.tools import files, dbupgrade -import ConfigParser import argparse import cherrypy import json @@ -20,6 +19,8 @@ import os import pwd import shutil import socket +import six +from six.moves import configparser import subprocess import sys import time @@ -35,10 +36,10 @@ CACHEDIR = '/var/cache/ipsilon' WSGI_SOCKET_PREFIX = None -class ConfigurationError(StandardError): +class ConfigurationError(Exception): def __init__(self, message): - StandardError.__init__(self, message) + super(ConfigurationError, self).__init__(message) #Silence cherrypy logging to screen @@ -94,7 +95,7 @@ def install(plugins, args): if os.path.exists(idp_conf): shutil.move(idp_conf, '%s.backup.%s' % (idp_conf, now)) if not os.path.exists(instance_conf): - os.makedirs(instance_conf, 0700) + os.makedirs(instance_conf, 0o700) confopts = {'instance': args['instance'], 'instanceurl': args['instanceurl'], 'needs_mount': args.get('needs_mount'), @@ -150,12 +151,12 @@ def install(plugins, args): if not os.path.exists(args['httpd_conf']): os.symlink(idp_conf, args['httpd_conf']) if not os.path.exists(args['public_data_dir']): - os.makedirs(args['public_data_dir'], 0755) + os.makedirs(args['public_data_dir'], 0o755) if not os.path.exists(args['wellknown_dir']): - os.makedirs(args['wellknown_dir'], 0755) + os.makedirs(args['wellknown_dir'], 0o755) sessdir = os.path.join(args['data_dir'], 'sessions') if not os.path.exists(sessdir): - os.makedirs(sessdir, 0700) + os.makedirs(sessdir, 0o700) data_conf = os.path.join(args['data_dir'], 'ipsilon.conf') if not os.path.exists(data_conf): os.symlink(ipsilon_conf, data_conf) @@ -265,10 +266,10 @@ def uninstall(plugins, args): data_dir = os.path.join(DATADIR, args['instance']) try: - tconf = ConfigParser.SafeConfigParser() + tconf = configparser.SafeConfigParser() tconf.read(os.path.join(instance_conf, 'ipsilon.conf')) cache_dir = tconf.get('global', 'cache_dir') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + except (configparser.NoOptionError, configparser.NoSectionError): cache_dir = None else: cache_dir = cache_dir.replace('"', '') @@ -351,7 +352,7 @@ def find_plugins(): def parse_config_profile(args): - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() files = config.read(args['config_profile']) if len(files) == 0: raise ConfigurationError('Config Profile file %s not found!' % @@ -501,7 +502,7 @@ if __name__ == '__main__': opts = parse_args(fplugins) logger.debug('Installation arguments:') - for k in sorted(opts.iterkeys()): + for k in sorted(six.iterkeys(opts)): logger.debug('%s: %s', k, opts[k]) if not opts['root_instance'] and opts['instance'] == 'root': diff --git a/ipsilon/ipsilon b/ipsilon/ipsilon index 1e775a2..1a63cfa 100755 --- a/ipsilon/ipsilon +++ b/ipsilon/ipsilon @@ -15,6 +15,7 @@ import os import atexit import string import cherrypy +import six from ipsilon import find_config from ipsilon.util.data import AdminStore from ipsilon.util import page @@ -70,7 +71,11 @@ template_env = Environment(loader=ChoiceLoader(template_loaders), extensions=['jinja2.ext.autoescape']) transchars = string.punctuation.replace('-', '').replace('.', '') -trans = string.maketrans(transchars, '_' * len(transchars)) +if six.PY2: + trans = string.maketrans(transchars, '_' * len(transchars)) +elif six.PY3: + trans = str.maketrans(transchars, '_' * len(transchars)) + if __name__ == "__main__": conf = {'global': {'server.socket_host': '0.0.0.0'}, '/': {'tools.staticdir.root': os.getcwd(), diff --git a/ipsilon/providers/openid/auth.py b/ipsilon/providers/openid/auth.py index 8bd907a..cfe45be 100644 --- a/ipsilon/providers/openid/auth.py +++ b/ipsilon/providers/openid/auth.py @@ -160,7 +160,11 @@ class AuthenticateRequest(ProviderPageBase): # We base64 encode the trust_root when looking up consent data to # ensure the client ID is safe for the cherrypy url routing - consentdata = user.get_consent('openid', b64encode(request.trust_root)) + trust_root = request.trust_root + if isinstance(trust_root, str): + trust_root = trust_root.encode('utf-8') + trust_root_b64 = b64encode(trust_root) + consentdata = user.get_consent('openid', trust_root_b64) if consentdata is not None: # Consent has already been granted self.debug('Consent already granted') @@ -183,10 +187,9 @@ class AuthenticateRequest(ProviderPageBase): # Store new consent consentdata = { - 'attributes': ad.keys() + 'attributes': list(ad.keys()) } - user.grant_consent('openid', b64encode(request.trust_root), - consentdata) + user.grant_consent('openid', trust_root_b64, consentdata) # all done we consent! return self._respond(self._response(request, us)) diff --git a/ipsilon/providers/openidc/admin.py b/ipsilon/providers/openidc/admin.py index b4a70d6..d30bd5a 100644 --- a/ipsilon/providers/openidc/admin.py +++ b/ipsilon/providers/openidc/admin.py @@ -14,6 +14,7 @@ from ipsilon.providers.openidc.provider import (Client, from copy import deepcopy import logging import re +import six INVALID_IN_CLIENT_ID = r'[^a-zA-Z0-9\-\.]' @@ -59,7 +60,7 @@ class ClientAdminPage(AdminPage): conf = self.client.get_config_obj() - for name, option in conf.iteritems(): + for name, option in six.iteritems(conf): if name in kwargs: value = kwargs[name] if isinstance(option, pconfig.List): diff --git a/ipsilon/providers/openidc/api.py b/ipsilon/providers/openidc/api.py index 11376db..0410027 100644 --- a/ipsilon/providers/openidc/api.py +++ b/ipsilon/providers/openidc/api.py @@ -20,7 +20,7 @@ class APIError(cherrypy.HTTPError, Log): response = {'error': error} if description: response['error_description'] = description - self._error_response = json.dumps(response) + self._error_response = json.dumps(response).encode('utf-8') cherrypy.response.headers.update({ 'Content-Type': 'application/json' }) @@ -137,6 +137,8 @@ class APIRequest(ProviderPageBase): 'client authentication error') self._set_apistore_key('api_client_authenticated', True) + if isinstance(client_id, bytes): + client_id = client_id.decode('utf-8') self._set_apistore_key('api_client_id', client_id) self._set_apistore_key('api_client', client) @@ -151,11 +153,11 @@ class APIRequest(ProviderPageBase): hdr = hdr[len('Basic '):] try: client_id, client_secret = \ - base64.b64decode(hdr).split(':', 1) + base64.b64decode(hdr).split(b':', 1) except Exception as e: # pylint: disable=broad-except self.error('Invalid request received: %s' % repr(e)) - self._respond_error('invalid_request', - 'invalid auth header') + return self._respond_error('invalid_request', + 'invalid auth header') self.debug('Client ID: %s' % client_id) self._handle_client_authentication('client_secret_basic', client_id, diff --git a/ipsilon/providers/openidc/auth.py b/ipsilon/providers/openidc/auth.py index 4817654..a8145bb 100644 --- a/ipsilon/providers/openidc/auth.py +++ b/ipsilon/providers/openidc/auth.py @@ -21,7 +21,7 @@ import hashlib import requests import time import json -import urllib +from six.moves.urllib.parse import urlencode URLROOT = 'openidc' @@ -101,7 +101,7 @@ class AuthenticateRequest(ProviderPageBase): else: url += '&' - url += urllib.urlencode(contents) + url += urlencode(contents) if response_mode in ['query', 'fragment', 'none']: raise cherrypy.HTTPRedirect(url) @@ -446,9 +446,9 @@ class Continue(AuthenticateRequest): # We are guaranteed that we either have a sector_identifier_uri # or that the hostpart of all redirect_uris are equal domain = get_url_hostpart(client['redirect_uris'][0]) - h.update(domain) - h.update(user.name) - h.update(self.cfg.idp_subject_salt) + h.update(domain.encode('utf-8')) + h.update(user.name.encode('utf-8')) + h.update(self.cfg.idp_subject_salt.encode('utf-8')) userinfo['sub'] = h.hexdigest() claims_userinfo = {} diff --git a/ipsilon/providers/openidc/provider.py b/ipsilon/providers/openidc/provider.py index 8685b5b..70202dc 100644 --- a/ipsilon/providers/openidc/provider.py +++ b/ipsilon/providers/openidc/provider.py @@ -9,7 +9,8 @@ from jwcrypto.jws import default_allowed_algs as jws_default_allowed_algs import json import time import requests -from urlparse import urlparse +import six +from six.moves.urllib.parse import urlparse def get_url_hostpart(url): @@ -45,10 +46,10 @@ class Registration(APIRequest): clt.generate_secret() except InvalidMetadata as ex: raise APIError(400, 'invalid_client_metadata', - ex.message) + str(ex)) except InvalidRedirectURI as ex: raise APIError(400, 'invalid_redirect_uri', - ex.message) + str(ex)) except pconfig.FieldValueError as ex: raise APIError(400, 'invalid_request', 'invalid field value for %s' % ex.field) @@ -117,7 +118,7 @@ class Client(pconfig.ConfigHelper): def generate_public(self): metadata = {} - for option, value in self.get_config_obj().iteritems(): + for option, value in six.iteritems(self.get_config_obj()): name = option.replace(' ', '_').lower() metadata[name] = value.get_value() return metadata diff --git a/ipsilon/providers/openidc/store.py b/ipsilon/providers/openidc/store.py index 9879fdc..86092b1 100644 --- a/ipsilon/providers/openidc/store.py +++ b/ipsilon/providers/openidc/store.py @@ -35,6 +35,8 @@ class OpenIDCStore(Store): data = {} for key in client: + if isinstance(client[key], bytes): + client[key] = client[key].decode('utf-8') data[key] = json.dumps(client[key]) client_id = self.new_unique_data('client', data) @@ -99,6 +101,8 @@ class OpenIDCStore(Store): return clients def getClient(self, client_id): + if isinstance(client_id, bytes): + client_id = client_id.decode('utf-8') if client_id.startswith('D-'): # This is a dynamically registered client ctype = 'dynamic' @@ -161,7 +165,7 @@ class OpenIDCStore(Store): return None if not return_expired and \ - datum['expires_at'] <= int(time.time()): + int(datum['expires_at']) <= int(time.time()): return None if expected_type and expected_type != 'Refresh' and \ diff --git a/ipsilon/providers/saml2/admin.py b/ipsilon/providers/saml2/admin.py index e9126e0..bb1ea20 100644 --- a/ipsilon/providers/saml2/admin.py +++ b/ipsilon/providers/saml2/admin.py @@ -17,7 +17,8 @@ from copy import deepcopy import requests import logging import base64 -from urlparse import urlparse +import six +from six.moves.urllib.parse import urlparse class NewSPAdminPage(AdminPage): @@ -60,7 +61,7 @@ class NewSPAdminPage(AdminPage): if ctype != 'multipart/form-data': self.debug("Invalid form type (%s), trying to cope" % ( cherrypy.request.content_type,)) - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): if key == 'name': name = value elif key == 'description': @@ -170,7 +171,7 @@ class SPAdminPage(AdminPage): conf = self.sp.get_config_obj() - for name, option in conf.iteritems(): + for name, option in six.iteritems(conf): if name in kwargs: value = kwargs[name] if isinstance(option, pconfig.List): @@ -220,7 +221,7 @@ class SPAdminPage(AdminPage): if len(new_db_values) != 0: try: # Validate user can make these changes - for (key, value) in new_db_values.iteritems(): + for (key, value) in six.iteritems(new_db_values): if key == 'Name': if (not self.user.is_admin and self.user.name != self.sp.owner): @@ -237,7 +238,7 @@ class SPAdminPage(AdminPage): ) # Make changes in current config - for name, option in conf.iteritems(): + for name, option in six.iteritems(conf): if name not in new_db_values: continue value = new_db_values.get(name, False) diff --git a/ipsilon/providers/saml2/auth.py b/ipsilon/providers/saml2/auth.py index 11d1bc9..45388f3 100644 --- a/ipsilon/providers/saml2/auth.py +++ b/ipsilon/providers/saml2/auth.py @@ -290,9 +290,9 @@ class AuthenticateRequest(ProviderPageBase): "idp nameid salt is not set in configuration" ) value = hashlib.sha512() - value.update(idpsalt) - value.update(login.remoteProviderId) - value.update(mappedattrs.get('_username')) + value.update(idpsalt.encode('utf-8')) + value.update(login.remoteProviderId.encode('utf-8')) + value.update(mappedattrs.get('_username').encode('utf-8')) nameid = '_' + value.hexdigest() elif nameidfmt == lasso.SAML2_NAME_IDENTIFIER_FORMAT_TRANSIENT: nameid = '_' + uuid.uuid4().hex @@ -345,6 +345,8 @@ class AuthenticateRequest(ProviderPageBase): self.log('Ignoring None value for attribute %s' % key) continue self.debug('value %s' % value) + if isinstance(value, bytes): + value = value.decode('utf-8') node = lasso.MiscTextNode.newWithString(value) node.textChild = True attrvalue = lasso.Saml2AttributeValue() diff --git a/ipsilon/providers/saml2/provider.py b/ipsilon/providers/saml2/provider.py index e913af3..adbeb19 100644 --- a/ipsilon/providers/saml2/provider.py +++ b/ipsilon/providers/saml2/provider.py @@ -62,7 +62,7 @@ def validate_sp_metadata(metadata): if len(newsps) != 1: raise InvalidProviderMetadata("Metadata must contain one Provider") - spid = newsps.keys()[0] + spid = list(newsps.keys())[0] return spid @@ -78,13 +78,13 @@ class ServiceProvider(ServiceProviderConfig): data = self.cfg.get_data(name='id', value=provider_id) if len(data) != 1: raise InvalidProviderId('multiple matches') - idval = data.keys()[0] + idval = list(data.keys())[0] data = self.cfg.get_data(idval=idval) self._properties = data[idval] self._staging = dict() self.load_config() self.logout_mechs = [] - xmldoc = etree.XML(str(data[idval]['metadata'])) + xmldoc = etree.XML(data[idval]['metadata'].encode('utf-8')) logout = xmldoc.xpath('//md:EntityDescriptor' '/md:SPSSODescriptor' '/md:SingleLogoutService', @@ -291,7 +291,7 @@ class ServiceProvider(ServiceProviderConfig): data = self.cfg.get_data(name='id', value=self.provider_id) if len(data) != 1: raise InvalidProviderId('Could not find SP data') - idval = data.keys()[0] + idval = list(data.keys())[0] data = dict() data[idval] = self._staging self.cfg.save_data(data) @@ -343,7 +343,7 @@ class ServiceProvider(ServiceProviderConfig): data = self.cfg.get_data(name='id', value=self.provider_id) if len(data) != 1: raise InvalidProviderId('Could not find SP data') - idval = data.keys()[0] + idval = list(data.keys())[0] self.cfg.del_datum(idval) def normalize_username(self, username): @@ -397,7 +397,7 @@ class ServiceProviderCreator(object): data = self.cfg.get_data(name='id', value=spid) if len(data) != 1: raise InvalidProviderId("Internal Error") - idval = data.keys()[0] + idval = list(data.keys())[0] data = self.cfg.get_data(idval=idval) sp = data[idval] self.cfg.idp.add_provider(sp) diff --git a/ipsilon/providers/saml2/rest.py b/ipsilon/providers/saml2/rest.py index d4d97fe..a348c6f 100644 --- a/ipsilon/providers/saml2/rest.py +++ b/ipsilon/providers/saml2/rest.py @@ -65,7 +65,7 @@ class SPS(RestProviderBase): data = idp.get_data(name='name', value=instance) if len(data) == 0: return rest_error(404, 'Provider %s not found' % instance) - idval = data.keys()[0] + idval = list(data.keys())[0] data = idp.get_data(idval=idval) else: data = idp.get_data() diff --git a/ipsilon/providers/saml2/sessions.py b/ipsilon/providers/saml2/sessions.py index 2cdf83f..fc3c68c 100644 --- a/ipsilon/providers/saml2/sessions.py +++ b/ipsilon/providers/saml2/sessions.py @@ -170,10 +170,15 @@ class SAMLSessionFactory(Log): session_ids = [] for c in candidates: - key = c.keys()[0] + key = list(c.keys())[0] if c[key].get('provider_id') == provider_id: samlsession = self._data_to_samlsession(key, c[key]) - session_ids.append(samlsession.session_id.encode('utf-8')) + sesid = samlsession.session_id + if not isinstance(sesid, str): + # In py3, we already has an str instance, in py2 we need to + # encode it + sesid = sesid.encode('utf-8') + session_ids.append(sesid) return tuple(session_ids) @@ -239,7 +244,7 @@ class SAMLSessionFactory(Log): for mech in logout_mechs: for c in candidates: - key = c.keys()[0] + key = list(c.keys())[0] if ((int(c[key].get('logoutstate', 0)) == LOGGED_IN) and (mech in c[key].get('supported_logout_mechs'))): samlsession = self._data_to_samlsession(key, c[key]) @@ -260,7 +265,7 @@ class SAMLSessionFactory(Log): # it's the "right" one if multiple logouts are started # at the same time from different SPs? for c in candidates: - key = c.keys()[0] + key = list(c.keys())[0] if int(c[key].get('logoutstate', 0)) == INIT_LOGOUT: samlsession = self._data_to_samlsession(key, c[key]) return samlsession @@ -277,7 +282,7 @@ class SAMLSessionFactory(Log): count = 0 for c in candidates: - key = c.keys()[0] + key = list(c.keys())[0] samlsession = self._data_to_samlsession(key, c[key]) self.debug('session %d: %s' % (count, samlsession.convert())) count += 1 diff --git a/ipsilon/tools/certs.py b/ipsilon/tools/certs.py index 9f2c42b..39cdeab 100644 --- a/ipsilon/tools/certs.py +++ b/ipsilon/tools/certs.py @@ -2,7 +2,6 @@ from subprocess import Popen import os -import string class Certificate(object): @@ -49,4 +48,4 @@ class Certificate(object): if cert[-1] == '-----END CERTIFICATE-----\n': cert = cert[:-1] - return string.join(cert) + return " ".join(cert) diff --git a/ipsilon/tools/saml2metadata.py b/ipsilon/tools/saml2metadata.py index ada2381..3b792b6 100755 --- a/ipsilon/tools/saml2metadata.py +++ b/ipsilon/tools/saml2metadata.py @@ -6,6 +6,7 @@ import datetime from ipsilon.tools.certs import Certificate from lxml import etree import lasso +import six SAML2_NAMEID_MAP = { @@ -125,7 +126,7 @@ class Metadata(object): svc = mdElement(self.role, service[0]) svc.set('Binding', service[1]) svc.set('Location', location) - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): svc.set(key, value) def add_allowed_name_format(self, name_format): @@ -138,7 +139,7 @@ class Metadata(object): if path is None: return data else: - with open(path, 'w') as f: + with open(path, 'wb') as f: f.write(data) diff --git a/ipsilon/util/config.py b/ipsilon/util/config.py index 365003d..bbc171a 100644 --- a/ipsilon/util/config.py +++ b/ipsilon/util/config.py @@ -7,6 +7,7 @@ import base64 import imghdr import hashlib import cherrypy +import six def name_from_image(image): @@ -263,7 +264,7 @@ class Image(Option): return None if not os.path.exists(self.__filename()): - with open(self.__filename(), 'w') as imagefile: + with open(self.__filename(), 'wb') as imagefile: imagefile.write(base64.b64decode(self._image)) @@ -511,7 +512,7 @@ class ConfigHelper(Log): if not self._config: raise AttributeError('Config not initialized, cannot import') - for key, value in config.iteritems(): + for key, value in six.iteritems(config): if key in self._config: self._config[key].import_value(str(value)) diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index 1741395..58f5ba3 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -9,7 +9,6 @@ from sqlalchemy.pool import QueuePool, SingletonThreadPool from sqlalchemy.schema import (PrimaryKeyConstraint, Index, AddConstraint, CreateIndex) from sqlalchemy.sql import select, and_ -import ConfigParser try: import etcd except ImportError: @@ -17,7 +16,9 @@ except ImportError: import os import json import uuid -from urlparse import urlparse +from six import text_type, binary_type +from six.moves import configparser +from six.moves.urllib.parse import urlparse import logging import time @@ -293,7 +294,7 @@ class FileStore(BaseStore): raise timestamp = stat.st_mtime if self._config is None or timestamp > self._timestamp: - self._config = ConfigParser.RawConfigParser() + self._config = configparser.RawConfigParser() self._config.optionxform = str self._config.read(self._filename) self._timestamp = timestamp @@ -1303,7 +1304,7 @@ class SAML2SessionStore(Store): return None elif count != 1: raise ValueError("Multiple entries returned") - return data.keys()[0] + return list(data.keys())[0] def get_data(self, idval=None, name=None, value=None): return self.get_unique_data(self.table, idval, name, value) @@ -1314,8 +1315,8 @@ class SAML2SessionStore(Store): datum['supported_logout_mechs'] ) for attr in datum: - if isinstance(datum[attr], str): - datum[attr] = unicode(datum[attr], 'utf-8') + if isinstance(datum[attr], binary_type): + datum[attr] = text_type(datum[attr], 'utf-8') return self.new_unique_data(self.table, datum, ttl) def get_session(self, session_id=None, request_id=None): @@ -1348,8 +1349,8 @@ class SAML2SessionStore(Store): def update_session(self, datum): for attr in datum: - if isinstance(datum[attr], str): - datum[attr] = unicode(datum[attr], 'utf-8') + if isinstance(datum[attr], binary_type): + datum[attr] = text_type(datum[attr], 'utf-8') self.save_unique_data(self.table, datum) def remove_session(self, uuidval): diff --git a/ipsilon/util/endpoint.py b/ipsilon/util/endpoint.py index 95d80d9..53a5e54 100644 --- a/ipsilon/util/endpoint.py +++ b/ipsilon/util/endpoint.py @@ -3,13 +3,8 @@ import cherrypy from ipsilon.util.log import Log from ipsilon.util.user import UserSession -from urllib import unquote +from six.moves.urllib.parse import unquote, urlparse from functools import wraps -try: - from urlparse import urlparse -except ImportError: - # pylint: disable=no-name-in-module, import-error - from urllib.parse import urlparse def allow_iframe(func): diff --git a/ipsilon/util/http.py b/ipsilon/util/http.py index fa9d725..63238ad 100644 --- a/ipsilon/util/http.py +++ b/ipsilon/util/http.py @@ -2,6 +2,7 @@ import cherrypy import fnmatch +from six import string_types def require_content_type(required=None, absent_ok=True, debug=False): @@ -36,7 +37,7 @@ def require_content_type(required=None, absent_ok=True, debug=False): if required is None: return - if isinstance(required, basestring): + if isinstance(required, string_types): required = [required] content_type = cherrypy.request.body.content_type.value diff --git a/ipsilon/util/log.py b/ipsilon/util/log.py index 165274d..9b36c73 100644 --- a/ipsilon/util/log.py +++ b/ipsilon/util/log.py @@ -1,11 +1,11 @@ # Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING import cherrypy -import cStringIO import inspect import os import traceback import logging +from six import StringIO def log_request_response(): @@ -95,7 +95,7 @@ def log_request_response(): original lines of text are indented. ''' - f = cStringIO.StringIO() + f = StringIO() lines = text.split('\n') @@ -120,7 +120,7 @@ def log_request_response(): each part of the multipart into a Part object containing information about the part and it's content. ''' - f = cStringIO.StringIO() + f = StringIO() f.write(indent_text('Name = %s\n' % part.name)) if part.headers: @@ -136,7 +136,7 @@ def log_request_response(): return string def print_param(name, value): - f = cStringIO.StringIO() + f = StringIO() # Might be a multipart Part object, if so format it if isinstance(value, cherrypy._cpreqbody.Part): # pylint:disable=W0212 @@ -176,7 +176,7 @@ def log_request_response(): as the new body contents for remainder of the processing pipeline to act upon (i.e. sent to the client) ''' - f = cStringIO.StringIO() + f = StringIO() for chunk in body: f.write(chunk) @@ -187,7 +187,7 @@ def log_request_response(): # --- End local functions --- - f = cStringIO.StringIO() + f = StringIO() request = cherrypy.serving.request remote = request.remote @@ -261,7 +261,7 @@ class Log(object): @staticmethod def stacktrace(): - buf = cStringIO.StringIO() + buf = StringIO() stack = traceback.extract_stack() traceback.print_list(stack[:-2], file=buf) diff --git a/ipsilon/util/page.py b/ipsilon/util/page.py index dbb3037..9ee2813 100644 --- a/ipsilon/util/page.py +++ b/ipsilon/util/page.py @@ -4,14 +4,7 @@ import cherrypy from ipsilon.util.endpoint import Endpoint from ipsilon.util.user import UserSession from ipsilon.util.trans import Transaction -from urllib import unquote -try: - from urlparse import urlparse - from urlparse import parse_qs -except ImportError: - # pylint: disable=no-name-in-module, import-error - from urllib.parse import urlparse - from urllib.parse import parse_qs +from six.moves.urllib.parse import unquote, urlparse, parse_qs def admin_protect(fn): diff --git a/ipsilon/util/security.py b/ipsilon/util/security.py index 5f5c38f..ca3fdc0 100644 --- a/ipsilon/util/security.py +++ b/ipsilon/util/security.py @@ -3,11 +3,16 @@ import base64 from cryptography.hazmat.primitives.constant_time import bytes_eq import os +from six import binary_type def generate_random_secure_string(size=32): - return base64.urlsafe_b64encode(os.urandom(size))[:size] + return base64.urlsafe_b64encode(os.urandom(size))[:size].decode('utf-8') def constant_time_string_comparison(stra, strb): - return bytes_eq(str(stra), str(strb)) + if not isinstance(stra, binary_type): + stra = stra.encode('utf-8') + if not isinstance(strb, binary_type): + strb = strb.encode('utf-8') + return bytes_eq(stra, strb) diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py index 1b723e8..065ca83 100644 --- a/ipsilon/util/sessions.py +++ b/ipsilon/util/sessions.py @@ -80,7 +80,7 @@ class SqlSession(Session): result = q.select({'id': self.id}) r = result.fetchone() if r: - data = str(base64.b64decode(r[1])) + data = base64.b64decode(r[1]).decode('utf-8') if not data.startswith('['): # This is a pre-upgrade pickle'd session. Just invalidate. self._delete() @@ -94,8 +94,10 @@ class SqlSession(Session): q = SqlQuery(self._db, 'sessions', SESSION_TABLE, trans=True) with q: q.delete({'id': self.id}) - data = json.dumps((self._data, expiration_time)) - q.insert((self.id, base64.b64encode(data), expiration_time)) + data = json.dumps((self._data, expiration_time)).encode('utf-8') + q.insert((self.id, + base64.b64encode(data).decode('utf-8'), + expiration_time)) def _delete(self): q = SqlQuery(self._db, 'sessions', SESSION_TABLE) diff --git a/ipsilon/util/webfinger.py b/ipsilon/util/webfinger.py index f551f2d..a0f0dc5 100644 --- a/ipsilon/util/webfinger.py +++ b/ipsilon/util/webfinger.py @@ -6,6 +6,7 @@ from ipsilon.util.log import Log from ipsilon.util.endpoint import allow_iframe import json +from six import string_types class WebFinger(Page, Log): @@ -34,7 +35,7 @@ class WebFinger(Page, Log): if 'rel' in kwargs: rels = kwargs['rel'] - if isinstance(rels, basestring): + if isinstance(rels, string_types): rels = [rels] else: rels = self.supported_rels.keys() diff --git a/tests/dbupgrades.py b/tests/dbupgrades.py index b5a8046..ce78712 100755 --- a/tests/dbupgrades.py +++ b/tests/dbupgrades.py @@ -108,7 +108,8 @@ class IpsilonTest(IpsilonTestBase): stderr=self.stderr) # Upgrade that database - cmd = [os.path.join(self.rootdir, + cmd = [self.pycmd, + os.path.join(self.rootdir, 'ipsilon/install/ipsilon-upgrade-database'), cfgfile] subprocess.check_call(cmd, @@ -121,16 +122,16 @@ class IpsilonTest(IpsilonTestBase): # Check all features in a newly created database # Let's verify if at least one index was created output = self.dump_db(db_outdir, with_readonly) - if 'CREATE INDEX' not in output: + if b'CREATE INDEX' not in output: raise Exception('Database upgrade did not introduce index') - if 'PRIMARY KEY' not in output: + if b'PRIMARY KEY' not in output: raise Exception('Database upgrade did not introduce primary ' + 'key') elif old_version == 1: # In 1 -> 2, we added indexes and primary keys # Let's verify if at least one index was created output = self.dump_db(db_outdir, with_readonly) - if 'CREATE INDEX' not in output: + if b'CREATE INDEX' not in output: raise Exception('Database upgrade did not introduce index') # SQLite did not support creating primary keys, so we can't test @@ -138,7 +139,7 @@ class IpsilonTest(IpsilonTestBase): # Version 3 added the authz_config table # Make sure it exists output = self.dump_db(db_outdir, with_readonly) - if 'TABLE authz_config' not in output: + if b'TABLE authz_config' not in output: raise Exception('Database upgrade did not introduce ' + 'authz_config table') @@ -149,7 +150,7 @@ class IpsilonTest(IpsilonTestBase): exe = self.execname if exe.endswith('c'): exe = exe[:-1] - exe = [exe] + exe = [self.pycmd, exe] exe.append(str(old_version)) if with_readonly: exe.append('readonly') diff --git a/tests/fconf.py b/tests/fconf.py index 439a0af..4e54c04 100755 --- a/tests/fconf.py +++ b/tests/fconf.py @@ -5,7 +5,7 @@ from helpers.common import IpsilonTestBase # pylint: disable=relative-import from helpers.control import TC # pylint: disable=relative-import from helpers.http import HttpSessions # pylint: disable=relative-import -import ConfigParser +from six.moves import configparser import os import pwd from string import Template @@ -122,7 +122,7 @@ def fixup_idp_conf(testdir): f.write(text) ipsilonconf = os.path.join(testdir, 'etc', idpname, 'ipsilon.conf') - newconf = ConfigParser.ConfigParser() + newconf = configparser.ConfigParser() with open(ipsilonconf, 'r') as f: newconf.readfp(f) with open(ipsilonconf, 'w+') as f: @@ -152,7 +152,8 @@ class IpsilonTest(IpsilonTestBase): self.setup_step("Testing database upgrade") cfgfile = os.path.join(self.testdir, 'etc', idpname, 'ipsilon.conf') - cmd = [os.path.join(self.rootdir, + cmd = [self.pycmd, + os.path.join(self.rootdir, 'ipsilon/install/ipsilon-upgrade-database'), cfgfile] subprocess.check_call(cmd, diff --git a/tests/helpers/common.py b/tests/helpers/common.py index 3adce6b..21d75b3 100755 --- a/tests/helpers/common.py +++ b/tests/helpers/common.py @@ -2,17 +2,18 @@ # # Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING -import ConfigParser +from six.moves import configparser import io import os import pwd import shutil import signal +import six import random from string import Template import subprocess -from control import TC # pylint: disable=relative-import +from .control import TC WRAP_HOSTNAME = 'idp.ipsilon.dev' @@ -92,6 +93,10 @@ class IpsilonTestBase(object): def set_py3(self, use_py3): self.py3 = use_py3 + @property + def pycmd(self): + return 'python3' if self.py3 else 'python' + def platform_supported(self): """This return whether the current platform supports this test. @@ -106,7 +111,7 @@ class IpsilonTestBase(object): return None def force_remove(self, op, name, info): - os.chmod(name, 0700) + os.chmod(name, 0o700) os.remove(name) def setup_base(self, path, test): @@ -161,7 +166,7 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, nameid='unspecified'): args_opts['port'] = port - newconf = ConfigParser.ConfigParser() + newconf = configparser.RawConfigParser() newconf.add_section('globals') for k in global_opts: newconf.set('globals', k, global_opts[k]) @@ -169,7 +174,10 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, for k in args_opts: newconf.set('arguments', k, args_opts[k]) - profile = io.BytesIO() + if six.PY2: + profile = io.BytesIO() + elif six.PY3: + profile = io.StringIO() newconf.write(profile) t = Template(profile.getvalue()) @@ -181,7 +189,7 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, 'TEST_USER': self.testuser}) filename = os.path.join(self.testdir, '%s_profile.cfg' % name) - with open(filename, 'wb') as f: + with open(filename, 'w') as f: f.write(text) return filename @@ -248,7 +256,8 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, env['LOGFILE'] = logfile else: env = {'LOGFILE': logfile} - cmd = [os.path.join(self.rootdir, + cmd = [self.pycmd, + os.path.join(self.rootdir, 'ipsilon/install/ipsilon-server-install'), '--config-profile=%s' % profile] subprocess.check_call(cmd, env=env, @@ -260,7 +269,8 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, def setup_sp_server(self, profile, name, addr, port, env): http_conf_file = self.setup_http(name, addr, port) - cmd = [os.path.join(self.rootdir, + cmd = [self.pycmd, + os.path.join(self.rootdir, 'ipsilon/install/ipsilon-client-install'), '--config-profile=%s' % profile] subprocess.check_call(cmd, env=env, diff --git a/tests/helpers/control.py b/tests/helpers/control.py index 9fb6c06..001e279 100644 --- a/tests/helpers/control.py +++ b/tests/helpers/control.py @@ -80,8 +80,8 @@ class TC(object): case done: ('done',) case fail: ('fail', 'some error') """ - if line.startswith(TC.prefix): - return tuple(line[len(TC.prefix):].split(':')) + if line.startswith(TC.prefix.encode('utf-8')): + return tuple(line[len(TC.prefix):].split(b':')) else: return None diff --git a/tests/helpers/http.py b/tests/helpers/http.py index 58b0b3e..3600ca9 100755 --- a/tests/helpers/http.py +++ b/tests/helpers/http.py @@ -5,9 +5,8 @@ from lxml import html import requests import string -import urlparse +from six.moves.urllib.parse import urlparse, urlencode import json -from urllib import urlencode from requests_kerberos import HTTPKerberosAuth, OPTIONAL @@ -103,7 +102,7 @@ class HttpSessions(object): return session.post(url, allow_redirects=False, **kwargs) def access(self, action, url, krb=False, **kwargs): - action = string.lower(action) + action = action.lower() if action == 'get': return self.get(url, krb, **kwargs) elif action == 'post': @@ -113,7 +112,7 @@ class HttpSessions(object): def new_url(self, referer, action): if action.startswith('/'): - u = urlparse.urlparse(referer) + u = urlparse(referer) return '%s://%s%s' % (u.scheme, u.netloc, action) return action diff --git a/tests/openidc.py b/tests/openidc.py index 1225046..edc224b 100755 --- a/tests/openidc.py +++ b/tests/openidc.py @@ -252,9 +252,9 @@ if __name__ == '__main__': page = sess.fetch_page(idpname, 'https://127.0.0.11:45081/sp/', require_consent=True) h = hashlib.sha256() - h.update('127.0.0.11') - h.update(user) - h.update('testcase') + h.update(b'127.0.0.11') + h.update(user.encode('utf-8')) + h.update(b'testcase') expect = { 'sub': h.hexdigest(), 'iss': 'https://127.0.0.10:45080/idp1/openidc/', @@ -278,9 +278,9 @@ if __name__ == '__main__': 'out=https%3A%2F%2F127.0.0.11%3A45081%2Fsp%2F', require_consent=True) h = hashlib.sha256() - h.update('127.0.0.11') - h.update(user) - h.update('testcase') + h.update(b'127.0.0.11') + h.update(user.encode('utf-8')) + h.update(b'testcase') expect = { 'sub': h.hexdigest(), 'iss': 'https://127.0.0.10:45080/idp1/openidc/', @@ -443,9 +443,9 @@ if __name__ == '__main__': if 'sub' not in info: raise Exception('No sub claim provided') h = hashlib.sha256() - h.update('127.0.0.11') - h.update(user) - h.update('testcase') + h.update(b'127.0.0.11') + h.update(user.encode('utf-8')) + h.update(b'testcase') if info['sub'] != h.hexdigest(): raise Exception('Sub claim invalid') @@ -462,9 +462,9 @@ if __name__ == '__main__': with TC.case('Access third SP Protected Area'): page = sess.fetch_page(idpname, 'https://127.0.0.13:45083/sp/') h = hashlib.sha256() - h.update('127.0.0.13') - h.update(user) - h.update('testcase') + h.update(b'127.0.0.13') + h.update(user.encode('utf-8')) + h.update(b'testcase') expect = { 'sub': h.hexdigest(), 'iss': 'https://127.0.0.10:45080/idp1/openidc/', diff --git a/tests/pgdb.py b/tests/pgdb.py index 11427c4..8e35592 100755 --- a/tests/pgdb.py +++ b/tests/pgdb.py @@ -7,6 +7,7 @@ from helpers.control import TC # pylint: disable=relative-import from helpers.http import HttpSessions # pylint: disable=relative-import import os import pwd +from six.moves import xrange from string import Template diff --git a/tests/testetcd.py b/tests/testetcd.py index 00e0688..015fc92 100755 --- a/tests/testetcd.py +++ b/tests/testetcd.py @@ -120,7 +120,7 @@ class IpsilonTest(IpsilonTestBase): if not p.wait() == 0: return 'No etcd installed' # Example line: etcd Version: 3.0.13 - if int(stdout.split('\n')[0].split(': ')[1][0]) < 3: + if int(stdout.split(b'\n')[0].split(b': ')[1][0]) < 3: return 'Etcd version < 3.0' try: import etcd # pylint: disable=unused-variable,import-error diff --git a/tests/testnameid.py b/tests/testnameid.py index 39bdac0..ce76a07 100755 --- a/tests/testnameid.py +++ b/tests/testnameid.py @@ -11,6 +11,7 @@ from ipsilon.tools.saml2metadata import SAML2_NAMEID_MAP import os import pwd import re +from six.moves import xrange from string import Template diff --git a/tests/tests.py b/tests/tests.py index a9cc4e8..f5543dc 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -131,6 +131,7 @@ def run_test(testname, test, args): if code: return (TEST_RESULT_FAIL, code, results) except Exception as e: # pylint: disable=broad-except + raise if post_setup: return (TEST_RESULT_EXCEPTION, e, results) else: From 27e2ef4c6e1565ae164c4d383a209815afbd21ca Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 10/17] Make tests default to verbosity 0 Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/tests.py b/tests/tests.py index f5543dc..16284b5 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -54,7 +54,7 @@ def parse_args(): help='List all available tests') parser.add_argument('--no-overview', '-q', action='store_true', help='Suppress final summary') - parser.add_argument('--verbose', '-v', action='count', + parser.add_argument('--verbose', '-v', action='count', default=0, help='Increase verbosity') parser.add_argument('--wrappers', default='auto', choices=['yes', 'no', 'auto'], From 239d44a78daefe818d33537892b4f1d4f4f63d59 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 11/17] Ignore _cp_config Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/providers/openidc/admin.py b/ipsilon/providers/openidc/admin.py index d30bd5a..201a11e 100644 --- a/ipsilon/providers/openidc/admin.py +++ b/ipsilon/providers/openidc/admin.py @@ -203,6 +203,8 @@ class DynamicAdminPage(AdminPage): unknown_client.exposed = True def __getattr__(self, attr): + if attr == '_cp_config': + raise AttributeError('No attribute %s' % attr) client = self.main.cfg.datastore.getClient(attr) if client is None: return self.unknown_client() From bbc3273ec4331f99809a8cb02386cf463e0a1ffb Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 12/17] Downgrade to older mod_auth_openidc for f29 tests Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/containers/Dockerfile-rpm-py3 b/tests/containers/Dockerfile-rpm-py3 index 3b0e42d..60b2bad 100644 --- a/tests/containers/Dockerfile-rpm-py3 +++ b/tests/containers/Dockerfile-rpm-py3 @@ -5,3 +5,5 @@ RUN yum install -y python3-pylint python3-pep8 python3-openid python3-openid-tea python3-requests-kerberos python3-lesscpy python3-sssdconfig \ python3-jwcrypto python3-jwcrypto python3-six RUN yum erase -y python2 +# There's a double-free in the f29 version +RUN yum downgrade -y https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Everything/x86_64/os/Packages/m/mod_auth_openidc-1.8.10.1-7.fc28.x86_64.rpm From 898eaf41367cd9b6a9caed46a9f31409a3916585 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 13/17] Re-add make Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/containers/Dockerfile-rpm-py3 b/tests/containers/Dockerfile-rpm-py3 index 60b2bad..478ece5 100644 --- a/tests/containers/Dockerfile-rpm-py3 +++ b/tests/containers/Dockerfile-rpm-py3 @@ -5,5 +5,6 @@ RUN yum install -y python3-pylint python3-pep8 python3-openid python3-openid-tea python3-requests-kerberos python3-lesscpy python3-sssdconfig \ python3-jwcrypto python3-jwcrypto python3-six RUN yum erase -y python2 +RUN yum install -y make # There's a double-free in the f29 version RUN yum downgrade -y https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Everything/x86_64/os/Packages/m/mod_auth_openidc-1.8.10.1-7.fc28.x86_64.rpm From 4393b0d0ced908f33988aeaec152cd3b29e21f1e Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 14/17] Assume deps are correct for container tests Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/containers/Dockerfile-base b/tests/containers/Dockerfile-base index bb0da78..111d135 100644 --- a/tests/containers/Dockerfile-base +++ b/tests/containers/Dockerfile-base @@ -3,4 +3,4 @@ VOLUME ["/code"] RUN useradd testuser \ && chown testuser:testuser /code WORKDIR /code -ENTRYPOINT ["/usr/bin/make", "testdeps", "test"] +ENTRYPOINT ["/usr/bin/make", "test"] From f162804d853f02e613db8881474cf7f6c3599cdd Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 15/17] Split lp-test from test and allow py3-pep8 Signed-off-by: Patrick Uiterwijk --- diff --git a/Makefile b/Makefile index d054d1c..6a9e43c 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ all: testdeps lint pep8 test security testdeps: # Determine if test deps are installed # First, some binaries - which pylint-3 which pep8 which httpd which postgres @@ -27,7 +26,6 @@ testdeps: python -c 'import ldap' python -c 'import pam' python -c 'import fedora' - python -c 'import ipapython' python -c 'import jinja2' python -c 'import psycopg2' # And now everything else @@ -50,7 +48,7 @@ lint: pep8: # Check style consistency - pep8 ipsilon + pep8 ipsilon || python3-pep8 ipsilon security: # Run a static analyzer aimed at security (OpenStack Bandit) @@ -91,7 +89,7 @@ tests: echo "Testdir: $(TESTDIR)" ./runtests --path=$(TESTDIR) -test: lp-test unittests tests +test: unittests tests unittests: PYTHONPATH=./ ./ipsilon/tools/saml2metadata.py From 39a9ac80264bce5af176d1f40851c9cd3a332b77 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 16/17] Run py3 unittests Signed-off-by: Patrick Uiterwijk --- diff --git a/Makefile b/Makefile index 6a9e43c..7d6efd4 100644 --- a/Makefile +++ b/Makefile @@ -92,8 +92,8 @@ tests: test: unittests tests unittests: - PYTHONPATH=./ ./ipsilon/tools/saml2metadata.py - PYTHONPATH=./ python ./ipsilon/util/policy.py + which python && PYTHONPATH=./ python ./ipsilon/tools/saml2metadata.py || PYTHONPATH=./ python3 ./ipsilon/tools/saml2metadata.py + which python && PYTHONPATH=./ python ./ipsilon/util/policy.py || PYTHONPATH=./ python3 ./ipsilon/util/policy.py sdist: python setup.py sdist From aa89b1fdaaf343165c593fa4d1aa231eba69211f Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Jan 08 2019 22:04:40 +0000 Subject: [PATCH 17/17] Rename utils.http to httputils to avoid import conflicts Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/providers/saml2idp.py b/ipsilon/providers/saml2idp.py index fbc9d68..fcd0996 100644 --- a/ipsilon/providers/saml2idp.py +++ b/ipsilon/providers/saml2idp.py @@ -12,7 +12,7 @@ from ipsilon.providers.saml2.sessions import SAMLSessionFactory from ipsilon.tools.certs import Certificate from ipsilon.tools import saml2metadata as metadata from ipsilon.tools import files -from ipsilon.util.http import require_content_type +from ipsilon.util.httputils import require_content_type from ipsilon.util.constants import SOAP_MEDIA_TYPE, XML_MEDIA_TYPE from ipsilon.util.user import UserSession from ipsilon.util.plugin import PluginObject diff --git a/ipsilon/util/http.py b/ipsilon/util/http.py deleted file mode 100644 index 63238ad..0000000 --- a/ipsilon/util/http.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING - -import cherrypy -import fnmatch -from six import string_types - - -def require_content_type(required=None, absent_ok=True, debug=False): - '''CherryPy Tool that validates request Content-Type. - - This is a CherryPy Tool that checks the Content-Type in a request and - raises HTTP Error 415 "Unsupported Media Type" if it does not match. - - The tool accepts a glob style pattern or list of patterns (see fnmatch) - and verifies the Content-Type in the request matches at least one of - the patterns, if not a HTTP Error 415 "Unsupported Media Type" is raised. - - If absent_ok is False and if the request does not contain a - Content-Type header a HTTP Error 415 "Unsupported Media Type" is - raised. - - The tool may be deployed use any of the standard methods for - invoking CherryPy tools, for example as a decorator: - - @cherrypy.tools.require_content_type(required='text/xml') - def POST(self, *args, **kwargs): - pass - - :param required: May be a single string or a list of strings. Each - string is interpreted as a glob style pattern (see fnmatch). - The Content-Type must match at least one pattern. - - :param absent_ok: Boolean specifying if the Content-Type header - must be present or if it is OK to be absent. - - ''' - if required is None: - return - - if isinstance(required, string_types): - required = [required] - - content_type = cherrypy.request.body.content_type.value - pattern = None - match = False - if content_type: - for pattern in required: - if fnmatch.fnmatch(content_type, pattern): - match = True - break - else: - if absent_ok: - return - - if debug: - cherrypy.log('require_content_type: required=%s, absent_ok=%s ' - 'content_type=%s match=%s pattern=%s' % - required, absent_ok, content_type, match, pattern) - - if not match: - acceptable = ', '.join(['"%s"' % x for x in required]) - if content_type: - content_type = '"%s"' % content_type - else: - content_type = 'not specified' - message = ('Content-Type must match one of following patterns [%s], ' - 'but the Content-Type was %s' % - (acceptable, content_type)) - raise cherrypy.HTTPError(415, message=message) diff --git a/ipsilon/util/httputils.py b/ipsilon/util/httputils.py new file mode 100644 index 0000000..63238ad --- /dev/null +++ b/ipsilon/util/httputils.py @@ -0,0 +1,69 @@ +# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING + +import cherrypy +import fnmatch +from six import string_types + + +def require_content_type(required=None, absent_ok=True, debug=False): + '''CherryPy Tool that validates request Content-Type. + + This is a CherryPy Tool that checks the Content-Type in a request and + raises HTTP Error 415 "Unsupported Media Type" if it does not match. + + The tool accepts a glob style pattern or list of patterns (see fnmatch) + and verifies the Content-Type in the request matches at least one of + the patterns, if not a HTTP Error 415 "Unsupported Media Type" is raised. + + If absent_ok is False and if the request does not contain a + Content-Type header a HTTP Error 415 "Unsupported Media Type" is + raised. + + The tool may be deployed use any of the standard methods for + invoking CherryPy tools, for example as a decorator: + + @cherrypy.tools.require_content_type(required='text/xml') + def POST(self, *args, **kwargs): + pass + + :param required: May be a single string or a list of strings. Each + string is interpreted as a glob style pattern (see fnmatch). + The Content-Type must match at least one pattern. + + :param absent_ok: Boolean specifying if the Content-Type header + must be present or if it is OK to be absent. + + ''' + if required is None: + return + + if isinstance(required, string_types): + required = [required] + + content_type = cherrypy.request.body.content_type.value + pattern = None + match = False + if content_type: + for pattern in required: + if fnmatch.fnmatch(content_type, pattern): + match = True + break + else: + if absent_ok: + return + + if debug: + cherrypy.log('require_content_type: required=%s, absent_ok=%s ' + 'content_type=%s match=%s pattern=%s' % + required, absent_ok, content_type, match, pattern) + + if not match: + acceptable = ', '.join(['"%s"' % x for x in required]) + if content_type: + content_type = '"%s"' % content_type + else: + content_type = 'not specified' + message = ('Content-Type must match one of following patterns [%s], ' + 'but the Content-Type was %s' % + (acceptable, content_type)) + raise cherrypy.HTTPError(415, message=message)