From be836825d8c19db8dad6d0183ecf4aedd96610bc Mon Sep 17 00:00:00 2001 From: Tomas Mlcoch Date: Mar 15 2016 07:25:53 +0000 Subject: [PATCH 1/2] Move config processing from CLI to koji.read_config() This patch is dmach's patch [1] refactored according to mikem's comments [2]. [1] https://lists.fedoraproject.org/pipermail/buildsys/2015-August/004858.html [2] https://lists.fedoraproject.org/pipermail/buildsys/2015-October/004900.html Signed-off-by: Tomas Mlcoch --- diff --git a/cli/koji b/cli/koji index 8c63e2a..07ff762 100755 --- a/cli/koji +++ b/cli/koji @@ -195,78 +195,19 @@ def get_options(): list_commands() parser.error('Unknown command: %s' % args[0]) assert False + # load local config - defaults = { - 'server' : 'http://localhost/kojihub', - 'weburl' : 'http://localhost/koji', - 'topurl' : None, - 'pkgurl' : None, - 'topdir' : '/mnt/koji', - 'max_retries' : None, - 'retry_interval': None, - 'anon_retry' : None, - 'offline_retry' : None, - 'offline_retry_interval' : None, - 'keepalive' : True, - 'timeout' : None, - 'use_fast_upload': False, - 'poll_interval': 5, - 'krbservice': 'host', - 'cert': '~/.koji/client.crt', - 'ca': '', # FIXME: remove in next major release - 'serverca': '~/.koji/serverca.crt', - 'authtype': None - } - #note: later config files override earlier ones - configs = koji.config_directory_contents('/etc/koji.conf.d') - if os.access('/etc/koji.conf', os.F_OK): - configs.append('/etc/koji.conf') - if options.configFile: - fn = os.path.expanduser(options.configFile) - if os.path.isdir(fn): - contents = koji.config_directory_contents(fn) - if not contents: - parser.error("No config files found in directory: %s" % fn) - configs.extend(contents) - else: - if not os.access(fn, os.F_OK): - parser.error("No such file: %s" % fn) - configs.append(fn) - else: - user_config_dir = os.path.expanduser("~/.koji/config.d") - configs.extend(koji.config_directory_contents(user_config_dir)) - fn = os.path.expanduser("~/.koji/config") - if os.access(fn, os.F_OK): - configs.append(fn) - got_conf = False - for configFile in configs: - f = open(configFile) - config = ConfigParser.ConfigParser() - config.readfp(f) - f.close() - if config.has_section(options.profile): - got_conf = True - for name, value in config.items(options.profile): - #note the defaults dictionary also serves to indicate which - #options *can* be set via the config file. Such options should - #not have a default value set in the option parser. - if defaults.has_key(name): - if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'): - defaults[name] = config.getboolean(options.profile, name) - elif name in ('max_retries', 'retry_interval', - 'offline_retry_interval', 'poll_interval', 'timeout'): - try: - defaults[name] = int(value) - except ValueError: - parser.error("value for %s config option must be a valid integer" % name) - assert False - else: - defaults[name] = value - if configs and not got_conf: - warn("Warning: no configuration for profile name: %s" % options.profile) - for name, value in defaults.iteritems(): + try: + result = koji.read_config(options.profile, user_config=options.configFile) + except koji.ConfigurationError, e: + parser.error(e.args[0]) + assert False + + # update options according to local config + for name, value in result.iteritems(): if getattr(options, name, None) is None: setattr(options, name, value) + dir_opts = ('topdir', 'cert', 'serverca') for name in dir_opts: # expand paths here, so we don't have to worry about it later diff --git a/koji/__init__.py b/koji/__init__.py index d43b8ef..9684510 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -29,12 +29,14 @@ except ImportError: sys.stderr.flush() import base64 import datetime +import ConfigParser import errno from fnmatch import fnmatch import httplib import logging import logging.handlers from koji.util import md5_constructor +import optparse import os import os.path import pwd @@ -332,6 +334,10 @@ class ImportError(GenericError): """Raised when an import fails""" faultCode = 1020 +class ConfigurationError(GenericError): + """Raised when load of koji configuration fails""" + faultCode = 1021 + class MultiCallInProgress(object): """ Placeholder class to be returned by method calls when in the process of @@ -1462,6 +1468,96 @@ def config_directory_contents(dir_name): return configs +def read_config(profile_name, user_config=None): + config_defaults = { + 'server' : 'http://localhost/kojihub', + 'weburl' : 'http://localhost/koji', + 'topurl' : None, + 'pkgurl' : None, + 'topdir' : '/mnt/koji', + 'max_retries' : None, + 'retry_interval': None, + 'anon_retry' : None, + 'offline_retry' : None, + 'offline_retry_interval' : None, + 'keepalive' : True, + 'timeout' : None, + 'use_fast_upload': False, + 'poll_interval': 5, + 'krbservice': 'host', + 'cert': '~/.koji/client.crt', + 'ca': '', # FIXME: remove in next major release + 'serverca': '~/.koji/serverca.crt', + 'authtype': None + } + + result = config_defaults.copy() + + #note: later config files override earlier ones + + # /etc/koji.conf.d + configs = config_directory_contents('/etc/koji.conf.d') + + # /etc/koji.conf + if os.access('/etc/koji.conf', os.F_OK): + configs.append('/etc/koji.conf') + + # User specific configuration + if user_config: + # Config file specified on command line + fn = os.path.expanduser(user_config) + if os.path.isdir(fn): + # Specified config is a directory + contents = config_directory_contents(fn) + if not contents: + raise ConfigurationError("No config files found in directory: %s" % fn) + configs.extend(contents) + else: + # Specified config is a file + if not os.access(fn, os.F_OK): + raise ConfigurationError("No such file: %s" % fn) + configs.append(fn) + else: + # User config + user_config_dir = os.path.expanduser("~/.koji/config.d") + configs.extend(config_directory_contents(user_config_dir)) + fn = os.path.expanduser("~/.koji/config") + if os.access(fn, os.F_OK): + configs.append(fn) + + # Load the configs in a particular order + got_conf = False + for configFile in configs: + f = open(configFile) + config = ConfigParser.ConfigParser() + config.readfp(f) + f.close() + if config.has_section(profile_name): + got_conf = True + for name, value in config.items(profile_name): + #note the config_defaults dictionary also serves to indicate which + #options *can* be set via the config file. Such options should + #not have a default value set in the option parser. + if result.has_key(name): + if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'): + result[name] = config.getboolean(profile_name, name) + elif name in ('max_retries', 'retry_interval', + 'offline_retry_interval', 'poll_interval', 'timeout'): + try: + result[name] = int(value) + except ValueError: + raise ConfigurationError("value for %s config option must be a valid integer" % name) + else: + result[name] = value + + # Check if the specified profile had a config specified + if configs and not got_conf: + sys.stderr.write("Warning: no configuration for profile name: %s\n" % profile_name) + sys.stderr.flush() + + return result + + class PathInfo(object): # ASCII numbers and upper- and lower-case letter for use in tmpdir() ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)] From 4192fc755e9a19e62519d44176432904207dbfc0 Mon Sep 17 00:00:00 2001 From: Tomas Mlcoch Date: Mar 15 2016 07:25:53 +0000 Subject: [PATCH 2/2] Support for koji profiles. This patch is dmach's patch [1] refactored according to mikem's comments [2]. [1] https://lists.fedoraproject.org/pipermail/buildsys/2015-August/004857.html [2] https://lists.fedoraproject.org/pipermail/buildsys/2015-October/004900.html Signed-off-by: Tomas Mlcoch --- diff --git a/docs/profiles.rst b/docs/profiles.rst new file mode 100644 index 0000000..04c997a --- /dev/null +++ b/docs/profiles.rst @@ -0,0 +1,74 @@ +============= +Koji Profiles +============= +This document describes how to work with koji profiles. + + +Command Line Interface +====================== +Koji client allows connecting to multiple koji instances from CLI +by using profiles. The default profile is given by executable file name, +which is 'koji'. + +To change koji profile, you can: + + * run koji with --profile=$profile_name argument + * change executable file name by symlinking $profile_name -> koji + + +Configuration Files +=================== +Configuration files are located in following locations: + + * /etc/koji.conf + * /etc/koji.conf.d/\*.conf + * ~/.koji/config.d/\*.conf + * user-specified config + +Koji reads them, looking for [$profile_name] sections. + + +Using Koji Profiles in Python +============================= +Instead of using koji module directly, +get profile specific module by calling:: + + >>> mod = koji.get_profile_module($profile_name) + +This module is clone of koji module with additional +profile specific tweaks. + +Profile configuration is available via:: + + >>> mod.config + + +Example +------- + +Print configuration:: + + import koji + + fedora_koji = koji.get_profile_module("koji") + ppc_koji = koji.get_profile_module("ppc-koji") + + for i in (fedora_koji, ppc_koji): + print "PROFILE: %s" % i.config.profile + for key, value in sorted(i.config.__dict__.items()): + print " %s = %s" % (key, value) + print + + +Use ClientSession:: + + import koji + + koji_module = koji.get_profile_module("koji") + client = koji_module.ClientSession(koji_module.config.server) + print client.listTags() + + +TODO +==== +* consider using pyxdg for user config locations diff --git a/koji/__init__.py b/koji/__init__.py index 9684510..775f4d8 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -33,6 +33,7 @@ import ConfigParser import errno from fnmatch import fnmatch import httplib +import imp import logging import logging.handlers from koji.util import md5_constructor @@ -62,6 +63,8 @@ import xml.sax.handler from xmlrpclib import loads, dumps, Fault import zipfile +PROFILE_MODULES = {} # {module_name: module_instance} + def _(args): """Stub function for translation""" return args @@ -1558,6 +1561,49 @@ def read_config(profile_name, user_config=None): return result +def get_profile_module(profile_name, config=None): + """ + Create module for a koji instance. + Override profile specific module attributes: + * BASEDIR + * config + * pathinfo + + profile_name is str with name of the profile + config is instance of optparse.Values() + """ + global PROFILE_MODULES # Dict with loaded modules + + # If config is passed use it and don't load koji config files by yourself + if config is None: + result = read_config(profile_name) + config = optparse.Values(result) + + # Prepare module name + mod_name = "__%s__%s" % (__name__, profile_name) + + # Check if profile module exists and if so return it + if mod_name in PROFILE_MODULES: + return PROFILE_MODULES[mod_name] + + # Load current module under a new name + koji_module_loc = imp.find_module(__name__) + mod = imp.load_module(mod_name, + None, + koji_module_loc[1], + koji_module_loc[2]) + + # Tweak config of the new module + mod.config = config + mod.BASEDIR = config.topdir + mod.pathinfo.topdir = config.topdir + + # Be sure that get_profile_module is only called from main module + mod.get_profile_module = get_profile_module + + return mod + + class PathInfo(object): # ASCII numbers and upper- and lower-case letter for use in tmpdir() ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]