From ffaa43bacdee7fcb27892624455809b25493ed1a Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 1/11] wrap dicts and iterators with list --- diff --git a/cli/koji b/cli/koji index ca8631e..285bf03 100755 --- a/cli/koji +++ b/cli/koji @@ -142,7 +142,7 @@ categories = { def get_epilog_str(progname=None): if progname is None: progname = os.path.basename(sys.argv[0]) or 'koji' - categories_ordered=', '.join(sorted(['all'] + categories.keys())) + categories_ordered=', '.join(sorted(['all'] + list(categories.keys()))) epilog_str = ''' Try "%(progname)s --help" for help about global options Try "%(progname)s help" to get all available commands @@ -242,7 +242,7 @@ def get_options(): assert False # pragma: no cover # update options according to local config - for name, value in result.iteritems(): + for name, value in result.items(): if getattr(options, name, None) is None: setattr(options, name, value) @@ -407,7 +407,7 @@ def display_tasklist_status(tasks): open = 0 failed = 0 done = 0 - for task_id in tasks.keys(): + for task_id in list(tasks.keys()): status = tasks[task_id].info['state'] if status == koji.TASK_STATES['FAILED']: failed += 1 @@ -420,7 +420,7 @@ def display_tasklist_status(tasks): print(" %d free %d open %d done %d failed" % (free, open, done, failed)) def display_task_results(tasks): - for task in [task for task in tasks.values() if task.level == 0]: + for task in [task for task in list(tasks.values()) if task.level == 0]: state = task.info['state'] task_label = task.str() @@ -448,7 +448,7 @@ def watch_tasks(session,tasklist,quiet=False): tasks[task_id] = TaskWatcher(task_id,session,quiet=quiet) while True: all_done = True - for task_id,task in tasks.items(): + for task_id,task in list(tasks.items()): changed = task.update() if not task.is_done(): all_done = False @@ -461,7 +461,7 @@ def watch_tasks(session,tasklist,quiet=False): rv = 1 for child in session.getTaskChildren(task_id): child_id = child['id'] - if not child_id in tasks.keys(): + if not child_id in list(tasks.keys()): tasks[child_id] = TaskWatcher(child_id, session, task.level + 1, quiet=quiet) tasks[child_id].update() # If we found new children, go through the list again, @@ -479,7 +479,7 @@ def watch_tasks(session,tasklist,quiet=False): if tasks and not quiet: progname = os.path.basename(sys.argv[0]) or 'koji' tlist = ['%s: %s' % (t.str(), t.display_state(t.info)) - for t in tasks.values() if not t.is_done()] + for t in list(tasks.values()) if not t.is_done()] print( \ """Tasks still running. You can continue to watch with the '%s watch-task' command. Running Tasks: @@ -511,7 +511,7 @@ def watch_logs(session, tasklist, opts): output = list_task_output_all_volumes(session, task_id) # convert to list of (file, volume) files = [] - for filename, volumes in output.iteritems(): + for filename, volumes in output.items(): files += [(filename, volume) for volume in volumes] if opts.log: @@ -664,7 +664,7 @@ def handle_edit_host(options, session, args): activate_session(session) vals = {} - for key, val in subopts.__dict__.items(): + for key, val in list(subopts.__dict__.items()): if val is not None: vals[key] = val if 'arches' in vals: @@ -1166,9 +1166,9 @@ def handle_maven_build(options, session, args): section=build_opts.section) except ValueError, e: parser.error(e.args[0]) - opts = params.values()[0] + opts = list(params.values())[0] if opts.pop('type', 'maven') != 'maven': - parser.error(_("Section %s does not contain a maven-build config") % params.keys()[0]) + parser.error(_("Section %s does not contain a maven-build config") % list(params.keys())[0]) source = opts.pop('scmurl') else: source = args[1] @@ -1225,9 +1225,9 @@ def handle_wrapper_rpm(options, session, args): section=build_opts.section) except ValueError, e: parser.error(e.args[0]) - opts = params.values()[0] + opts = list(params.values())[0] if opts.get('type') != 'wrapper': - parser.error(_("Section %s does not contain a wrapper-rpm config") % params.keys()[0]) + parser.error(_("Section %s does not contain a wrapper-rpm config") % list(params.keys())[0]) url = opts['scmurl'] package = opts['buildrequires'][0] target_info = session.getBuildTarget(target, strict=True) @@ -1618,7 +1618,7 @@ def handle_import(options, session, args): nvr = "%(name)s-%(version)s-%(release)s" % koji.parse_NVRA(data['sourcerpm']) to_import.setdefault(nvr,[]).append((path,data)) builds_missing = False - nvrs = to_import.keys() + nvrs = list(to_import.keys()) nvrs.sort() for nvr in nvrs: to_import[nvr].sort() @@ -2080,7 +2080,7 @@ def handle_prune_signed_copies(options, session, args): #that the build was recently untagged from tags.setdefault(entry['tag_name'], 1) if options.debug: - print("Tags: %s" % tags.keys()) + print("Tags: %s" % list(tags.keys())) for tag_name in tags: if tag_name == options.trashcan_tag: if options.debug: @@ -2250,7 +2250,7 @@ def handle_prune_signed_copies(options, session, args): build_space = 0 if not by_sig and options.debug: print("(build has no signatures)") - for sigkey, rpms in by_sig.iteritems(): + for sigkey, rpms in by_sig.items(): mycount = 0 archdirs = {} sigdirs = {} @@ -2297,7 +2297,7 @@ def handle_prune_signed_copies(options, session, args): except OSError, e: print("Error removing %s: %s" % (signedpath, e)) if len(sigdirs) == 1: - dir = sigdirs.keys()[0] + dir = list(sigdirs.keys())[0] if options.test: print("Would have removed dir: %s" % dir) else: @@ -3610,35 +3610,35 @@ def handle_clone_tag(options, session, args): dstgroups[group['name']] = group #construct to-do lists. paddlist = [] # list containing new packages to be added from src tag - for (package_name, pkg) in srcpkgs.iteritems(): + for (package_name, pkg) in srcpkgs.items(): if package_name not in dstpkgs: paddlist.append(pkg) paddlist.sort(key = lambda x: x['package_name']) pdellist = [] # list containing packages no more present in dst tag - for (package_name, pkg) in dstpkgs.iteritems(): + for (package_name, pkg) in dstpkgs.items(): if package_name not in srcpkgs: pdellist.append(pkg) pdellist.sort(key = lambda x: x['package_name']) baddlist = [] # list containing new builds to be added from src tag - for (nvr, lbld) in srclblds.iteritems(): + for (nvr, lbld) in srclblds.items(): if nvr not in dstlblds: baddlist.append(lbld) baddlist.sort(key = lambda x: x['package_name']) bdellist = [] # list containing new builds to be removed from src tag - for (nvr, lbld) in dstlblds.iteritems(): + for (nvr, lbld) in dstlblds.items(): if nvr not in srclblds: bdellist.append(lbld) bdellist.sort(key = lambda x: x['package_name']) gaddlist = [] # list containing new groups to be added from src tag - for (grpname, group) in srcgroups.iteritems(): + for (grpname, group) in srcgroups.items(): if grpname not in dstgroups: gaddlist.append(group) gdellist = [] # list containing groups to be removed from src tag - for (grpname, group) in dstgroups.iteritems(): + for (grpname, group) in dstgroups.items(): if grpname not in srcgroups: gdellist.append(group) grpchanges = {} # dict of changes to make in shared groups - for (grpname, group) in srcgroups.iteritems(): + for (grpname, group) in srcgroups.items(): if grpname in dstgroups: grpchanges[grpname] = {'adds':[], 'dels':[]} # Store whether group is inherited or not @@ -4332,7 +4332,7 @@ def _print_histline(entry, **kwargs): else: return '%s.name' % key if edit: - keys = x.keys() + keys = list(x.keys()) keys.sort() y = other[-1] for key in keys: @@ -4347,7 +4347,7 @@ def _print_histline(entry, **kwargs): continue print(" %s: %s -> %s" % (key, x[key], y[key])) elif create and options.verbose and table != 'tag_listing': - keys = x.keys() + keys = list(x.keys()) keys.sort() # the table keys have already been represented in the base format string also_hidden = list(_table_keys[table]) @@ -4516,7 +4516,7 @@ def anon_handle_list_history(options, session, args): kwargs['afterEvent'] = last_event def _handleMap(lines, data, prefix=''): - for key, val in data.items(): + for key, val in list(data.items()): if key != '__starstar': lines.append(' %s%s: %s' % (prefix, key, val)) @@ -4590,7 +4590,7 @@ def _do_parseTaskParams(session, method, task_id): _handleOpts(lines, params[4]) elif method == 'chainmaven': lines.append("Builds:") - for package, opts in params[0].items(): + for package, opts in list(params[0].items()): lines.append(" " + package) _handleMap(lines, opts, prefix=" ") lines.append("Build Target: %s" % params[1]) @@ -4823,7 +4823,7 @@ def anon_handle_taginfo(options, session, args): print("Include all Maven archives?: %s" % (info['maven_include_all'] and 'yes' or 'no')) if 'extra' in info: print("Tag options:") - keys = info['extra'].keys() + keys = list(info['extra'].keys()) keys.sort() for key in keys: print(" %s : %s" % (key, pprint.pformat(info['extra'][key]))) @@ -7003,7 +7003,7 @@ def anon_handle_wait_repo(options, session, args): targets = session.getBuildTargets(destTagID=tag_info['id']) if targets: maybe = {}.fromkeys([t['build_tag_name'] for t in targets]) - maybe = maybe.keys() + maybe = list(maybe.keys()) maybe.sort() print("Suggested tags: %s" % ', '.join(maybe)) return 1 @@ -7428,7 +7428,7 @@ def handle_help(options, session, args): chosen = set(args) if options.admin: chosen.add('admin') - avail = set(categories.keys() + ['all']) + avail = set(list(categories.keys()) + ['all']) unavail = chosen - avail for arg in unavail: print("No such help category: %s" % arg) @@ -7441,13 +7441,13 @@ def handle_help(options, session, args): def list_commands(categories_chosen=None): if categories_chosen is None or "all" in categories_chosen: - categories_chosen = categories.keys() + categories_chosen = list(categories.keys()) else: # copy list since we're about to modify it categories_chosen = list(categories_chosen) categories_chosen.sort() handlers = [] - for name,value in globals().items(): + for name,value in list(globals().items()): if name.startswith('handle_'): alias = name.replace('handle_','') alias = alias.replace('_','-') diff --git a/koji/__init__.py b/koji/__init__.py index cf21abd..e8c934f 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -380,7 +380,7 @@ def convertFault(fault): code = getattr(fault, 'faultCode', None) if code is None: return fault - for v in globals().values(): + for v in list(globals().values()): if isinstance(v, type(Exception)) and issubclass(v, GenericError) and \ code == getattr(v, 'faultCode', None): ret = v(fault.faultString) @@ -398,7 +398,7 @@ def listFaults(): desc: the description of the exception (docstring) """ ret = [] - for n, v in globals().items(): + for n, v in list(globals().items()): if isinstance(v, type(Exception)) and issubclass(v, GenericError): code = getattr(v, 'faultCode', None) if code is None: @@ -627,11 +627,11 @@ class RawHeader(object): print("Store at offset %d (%0x)" % (store, store)) #sort entries by offset, dtype #also rearrange: tag, dtype, offset, count -> offset, dtype, tag, count - order = sorted([(x[2], x[1], x[0], x[3]) for x in self.index.itervalues()]) + order = sorted([(x[2], x[1], x[0], x[3]) for x in list(self.index.values())]) next = store #map some rpmtag codes tags = {} - for name, code in rpm.__dict__.iteritems(): + for name, code in list(rpm.__dict__.items()): if name.startswith('RPMTAG_') and isinstance(code, int): tags[code] = name[7:].lower() for entry in order: @@ -1106,7 +1106,7 @@ def parse_pom(path=None, contents=None): xml.sax.parseString(contents, handler) for field in fields: - if field not in values.keys(): + if field not in list(values.keys()): raise GenericError('could not extract %s from POM: %s' % (field, (path or ''))) return values @@ -1407,7 +1407,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) if opts.get('maven_opts'): mavenrc = 'export MAVEN_OPTS="%s"\n' % ' '.join(opts['maven_opts']) if opts.get('maven_envs'): - for name, val in opts['maven_envs'].iteritems(): + for name, val in list(opts['maven_envs'].items()): mavenrc += 'export %s="%s"\n' % (name, val) if mavenrc: files['etc/mavenrc'] = mavenrc @@ -1470,25 +1470,25 @@ name=build """ % locals()) parts.append("\n") - for key, value in config_opts.iteritems(): + for key, value in list(config_opts.items()): parts.append("config_opts[%r] = %r\n" % (key, value)) parts.append("\n") - for key, value in plugin_conf.iteritems(): + for key, value in list(plugin_conf.items()): parts.append("config_opts['plugin_conf'][%r] = %r\n" % (key, value)) parts.append("\n") if bind_opts: # This line is REQUIRED for mock to work if bind_opts defined. parts.append("config_opts['internal_dev_setup'] = False\n") - for key in bind_opts.keys(): - for mnt_src, mnt_dest in bind_opts.get(key).iteritems(): + for key in list(bind_opts.keys()): + for mnt_src, mnt_dest in list(bind_opts.get(key).items()): parts.append("config_opts['plugin_conf']['bind_mount_opts'][%r].append((%r, %r))\n" % (key, mnt_src, mnt_dest)) parts.append("\n") - for key, value in macros.iteritems(): + for key, value in list(macros.items()): parts.append("config_opts['macros'][%r] = %r\n" % (key, value)) parts.append("\n") - for key, value in files.iteritems(): + for key, value in list(files.items()): parts.append("config_opts['files'][%r] = %r\n" % (key, value)) return ''.join(parts) @@ -1517,7 +1517,7 @@ def format_exc_plus(): rv += "Frame %s in %s at line %s\n" % (frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno) - for key, value in frame.f_locals.items(): + for key, value in list(frame.f_locals.items()): rv += " %20s = " % key # we must _absolutely_ avoid propagating eceptions, and str(value) # COULD cause any exception, so we MUST catch any...: @@ -1728,7 +1728,7 @@ def get_profile_module(profile_name, config=None): class PathInfo(object): # ASCII numbers and upper- and lower-case letter for use in tmpdir() - ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)] + ASCII_CHARS = [chr(i) for i in list(range(48, 58)) + list(range(65, 91)) + list(range(97, 123))] def __init__(self, topdir=None): self._topdir = topdir @@ -2669,7 +2669,7 @@ class DBHandler(logging.Handler): values = [] data = {} record.message = record.getMessage() - for key, value in self.mapping.iteritems(): + for key, value in list(self.mapping.items()): value = str(value) if value.find("%(asctime)") >= 0: if self.formatter: diff --git a/koji/util.py b/koji/util.py index f3d867f..0dadd98 100644 --- a/koji/util.py +++ b/koji/util.py @@ -232,17 +232,17 @@ class LazyDict(dict): return LazyDict(self) def values(self): - return [lazy_eval(val) for val in super(LazyDict, self).values()] + return [lazy_eval(val) for val in list(super(LazyDict, self).values())] def items(self): - return [(key, lazy_eval(val)) for key, val in super(LazyDict, self).items()] + return [(key, lazy_eval(val)) for key, val in list(super(LazyDict, self).items())] def itervalues(self): - for val in super(LazyDict, self).itervalues(): + for val in super(LazyDict, self).values(): yield lazy_eval(val) def iteritems(self): - for key, val in super(LazyDict, self).iteritems(): + for key, val in super(LazyDict, self).items(): yield key, lazy_eval(val) def pop(self, key, *args, **kwargs): @@ -496,11 +496,11 @@ def tsort(parts): parts = parts.copy() result = [] while True: - level = set([name for name, deps in parts.iteritems() if not deps]) + level = set([name for name, deps in parts.items() if not deps]) if not level: break result.append(level) - parts = dict([(name, deps - level) for name, deps in parts.iteritems() + parts = dict([(name, deps - level) for name, deps in parts.items() if name not in level]) if parts: raise ValueError('total ordering not possible') @@ -641,7 +641,7 @@ def parse_maven_chain(confs, scratch=False): """ builds = parse_maven_params(confs, chain=True, scratch=scratch) depmap = {} - for package, params in builds.items(): + for package, params in list(builds.items()): depmap[package] = set(params.get('buildrequires', [])) try: tsort(depmap) From 605a5ca6233ad5041bfe01a1f13b6b0d22f96339 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 2/11] import python3 modules --- diff --git a/cli/koji b/cli/koji index 285bf03..cc1a919 100755 --- a/cli/koji +++ b/cli/koji @@ -40,14 +40,15 @@ except ImportError: # pragma: no cover import simplejson as json except ImportError: json = None -import ConfigParser +try: + import ConfigParser +except ImportError: # pragma: no cover + import configparser as ConfigParser import base64 import dateutil.parser -import errno import koji import koji.util import fnmatch -from koji.util import md5_constructor import logging import os import re @@ -58,9 +59,12 @@ import stat import string import time import traceback -import urlgrabber.grabber as grabber +import urlgrabber as grabber import urlgrabber.progress as progress -import xmlrpclib +try: + import xmlrpclib +except ImportError: + import xmlrpc.client as xmlrpclib try: import libcomps except ImportError: # pragma: no cover @@ -1924,7 +1928,7 @@ def handle_import_sig(options, session, args): previous = session.queryRPMSigs(rpm_id=rinfo['id'], sigkey=sigkey) assert len(previous) <= 1 if previous: - sighash = md5_constructor(sighdr).hexdigest() + sighash = koji.util.md5_constructor(sighdr).hexdigest() if previous[0]['sighash'] == sighash: print(_("Signature already imported: %s") % path) continue diff --git a/koji/__init__.py b/koji/__init__.py index e8c934f..837d504 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -29,16 +29,21 @@ except ImportError: # pragma: no cover sys.stderr.flush() import base64 import datetime -import ConfigParser +try: + import ConfigParser +except ImportError: # pragma: no cover + import configparser as ConfigParser import errno -import exceptions from fnmatch import fnmatch -import httplib +try: + import httplib +except ImportError: + import http.client as httplib import imp import logging import logging.handlers -from koji.util import md5_constructor SSL_Error = None +PythonImportError = ImportError # will be masked by koji.ImportError try: from OpenSSL.SSL import Error as SSL_Error except Exception: #pragma: no cover @@ -55,11 +60,11 @@ import random import re try: import requests -except ImportError: #pragma: no cover +except ImportError: # pragma: no cover requests = None try: from requests_kerberos import HTTPKerberosAuth -except ImportError: #pragma: no cover +except ImportError: # pragma: no cover HTTPKerberosAuth = None import rpm import shutil @@ -70,14 +75,31 @@ import tempfile import time import traceback import urllib -import urllib2 -import urlparse -import util +try: + import urllib2 + import urlparse + urllib_urlopen = urllib2.urlopen + urllib_splitport = urllib.splitport + urllib_urlencode = urllib.urlencode + urlparse_urlsplit = urlparse.urlsplit +except ImportError: + urllib_urlopen = urllib.request.urlopen + urllib_splitport = urllib.parse.splitport + urllib_urlencode = urllib.parse.urlencode + urlparse_urlsplit = urllib.parse.urlsplit import warnings -import xmlrpclib import xml.sax import xml.sax.handler -from xmlrpclib import loads, dumps, Fault +try: + import xmlrpclib + from xmlrpclib import loads, dumps, Fault +except: # pragma: no cover + import xmlrpc.client as xmlrpclib + loads = xmlrpclib.loads + dumps = xmlrpclib.dumps + Fault = xmlrpclib.Fault + +from koji.util import md5_constructor, adler32_constructor PROFILE_MODULES = {} # {module_name: module_instance} @@ -1534,7 +1556,7 @@ def openRemoteFile(relpath, topurl=None, topdir=None, tempdir=None): on options""" if topurl: url = "%s/%s" % (topurl, relpath) - src = urllib2.urlopen(url) + src = urllib_urlopen(url) fo = tempfile.TemporaryFile(dir=tempdir) shutil.copyfileobj(src, fo) src.close() @@ -2060,7 +2082,7 @@ class ClientSession(object): pass if not krbV: - raise exceptions.ImportError( + raise PythonImportError( "Please install python-krbV to use kerberos." ) @@ -2125,8 +2147,8 @@ class ClientSession(object): """Get the Kerberos principal of the server we're connecting to, based on baseurl.""" - uri = urlparse.urlsplit(self.baseurl) - host, port = urllib.splitport(uri[1]) + uri = urlparse_urlsplit(self.baseurl) + host, port = urllib_splitport(uri[1]) if self.opts.get('krb_rdns', True): servername = socket.getfqdn(host) else: @@ -2138,13 +2160,13 @@ class ClientSession(object): def gssapi_login(self, proxyuser=None): if not HTTPKerberosAuth: - raise exceptions.ImportError( + raise PythonImportError( "Please install python-requests-kerberos to use GSSAPI." ) # force https old_baseurl = self.baseurl - uri = urlparse.urlsplit(self.baseurl) + uri = urlparse_urlsplit(self.baseurl) if uri[0] != 'https': self.baseurl = 'https://%s%s' % (uri[1], uri[2]) @@ -2188,7 +2210,7 @@ class ClientSession(object): # when API is changed # force https - uri = urlparse.urlsplit(self.baseurl) + uri = urlparse_urlsplit(self.baseurl) if uri[0] != 'https': self.baseurl = 'https://%s%s' % (uri[1], uri[2]) @@ -2263,7 +2285,7 @@ class ClientSession(object): sinfo = self.sinfo.copy() sinfo['callnum'] = self.callnum self.callnum += 1 - handler = "%s?%s" % (self.baseurl, urllib.urlencode(sinfo)) + handler = "%s?%s" % (self.baseurl, urllib_urlencode(sinfo)) elif name == 'sslLogin': handler = self.baseurl + '/ssllogin' else: @@ -2471,7 +2493,7 @@ class ClientSession(object): if callback: callback(0, size, 0, 0, 0) problems = False - full_chksum = util.adler32_constructor() + full_chksum = adler32_constructor() # cycle is need to run at least once (for empty files) first_cycle = True callopts = {'overwrite': overwrite} @@ -2486,7 +2508,7 @@ class ClientSession(object): result = self._callMethod('rawUpload', (chunk, ofs, path, name), callopts) if self.retries > 1: problems = True - hexdigest = util.adler32_constructor(chunk).hexdigest() + hexdigest = adler32_constructor(chunk).hexdigest() full_chksum.update(chunk) if result['size'] != len(chunk): raise GenericError("server returned wrong chunk size: %s != %s" % (result['size'], len(chunk))) @@ -2534,7 +2556,7 @@ class ClientSession(object): args['volume'] = volume size = len(chunk) self.callnum += 1 - handler = "%s?%s" % (self.baseurl, urllib.urlencode(args)) + handler = "%s?%s" % (self.baseurl, urllib_urlencode(args)) headers = [ ('User-Agent', 'koji/1.7'), #XXX ("Content-Type", "application/octet-stream"), diff --git a/koji/util.py b/koji/util.py index 0dadd98..a7213df 100644 --- a/koji/util.py +++ b/koji/util.py @@ -30,7 +30,10 @@ import shutil import stat import sys import time -import ConfigParser +try: + import ConfigParser +except ImportError: # pragma: no cover + import configparser as ConfigParser from zlib import adler32 # imported from kojiweb and kojihub From 4fdfb50fc557adc9110951bdceb7fe0849f45f61 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 3/11] use python2/3 compatible try/except --- diff --git a/cli/koji b/cli/koji index cc1a919..e90a455 100755 --- a/cli/koji +++ b/cli/koji @@ -241,7 +241,8 @@ def get_options(): # load local config try: result = koji.read_config(options.profile, user_config=options.configFile) - except koji.ConfigurationError, e: + except koji.ConfigurationError: + e = sys.exc_info()[1] parser.error(e.args[0]) assert False # pragma: no cover @@ -343,8 +344,8 @@ class TaskWatcher(object): error = None try: result = self.session.getTaskResult(self.id) - except (xmlrpclib.Fault,koji.GenericError),e: - error = e + except (xmlrpclib.Fault, koji.GenericError): + error = sys.exc_info()[1] if error is None: # print("%s: complete" % self.str()) # We already reported this task as complete in update() @@ -551,7 +552,8 @@ def list_task_output_all_volumes(session, task_id): """List task output with all volumes, or fake it""" try: return session.listTaskOutput(task_id, all_volumes=True) - except koji.GenericError, e: + except koji.GenericError: + e = sys.exc_info()[1] if 'got an unexpected keyword argument' not in str(e): raise # otherwise leave off the option and fake it @@ -945,7 +947,7 @@ def _progress_callback(uploaded, total, piece, time, total_time): def _running_in_bg(): try: return (not os.isatty(0)) or (os.getpgrp() != os.tcgetpgrp(0)) - except OSError, e: + except OSError: return True def handle_build(options, session, args): @@ -1168,7 +1170,8 @@ def handle_maven_build(options, session, args): try: params = koji.util.parse_maven_param(build_opts.inis, scratch=build_opts.scratch, section=build_opts.section) - except ValueError, e: + except ValueError: + e = sys.exc_info()[1] parser.error(e.args[0]) opts = list(params.values())[0] if opts.pop('type', 'maven') != 'maven': @@ -1227,7 +1230,8 @@ def handle_wrapper_rpm(options, session, args): try: params = koji.util.parse_maven_param(build_opts.inis, scratch=build_opts.scratch, section=build_opts.section) - except ValueError, e: + except ValueError: + e = sys.exc_info()[1] parser.error(e.args[0]) opts = list(params.values())[0] if opts.get('type') != 'wrapper': @@ -1301,7 +1305,8 @@ def handle_maven_chain(options, session, args): opts[key] = val try: builds = koji.util.parse_maven_chain(args[1:], scratch=opts.get('scratch')) - except ValueError, e: + except ValueError: + e = sys.exc_info()[1] parser.error(e.args[0]) priority = None if build_opts.background: @@ -1669,7 +1674,8 @@ def handle_import(options, session, args): sys.stdout.flush() try: session.importRPM(serverdir, os.path.basename(path)) - except koji.GenericError, e: + except koji.GenericError: + e = sys.exc_info()[1] print(_("\nError importing: %s" % str(e).splitlines()[-1])) sys.stdout.flush() else: @@ -2278,7 +2284,8 @@ def handle_prune_signed_copies(options, session, args): print("Unlinking: %s" % signedpath) try: os.unlink(signedpath) - except OSError, e: + except OSError: + e = sys.exc_info()[1] print("Error removing %s: %s" % (signedpath, e)) print("This script needs write access to %s" % koji.BASEDIR) continue @@ -2298,7 +2305,8 @@ def handle_prune_signed_copies(options, session, args): print("Removing dir: %s" % dir) try: os.rmdir(dir) - except OSError, e: + except OSError: + e = sys.exc_info()[1] print("Error removing %s: %s" % (signedpath, e)) if len(sigdirs) == 1: dir = list(sigdirs.keys())[0] @@ -2309,7 +2317,8 @@ def handle_prune_signed_copies(options, session, args): print("Removing dir: %s" % dir) try: os.rmdir(dir) - except OSError, e: + except OSError: + e = sys.exc_info()[1] print("Error removing %s: %s" % (signedpath, e)) elif len(sigdirs) > 1: print("Warning: more than one signature dir for %s: %r" % (sigkey, sigdirs)) @@ -2543,7 +2552,8 @@ def handle_import_in_place(options, session, args): sys.stdout.write(_("importing %s... ") % nvr) try: session.importBuildInPlace(data) - except koji.GenericError, e: + except koji.GenericError: + e = sys.exc_info()[1] print(_("\nError importing: %s" % str(e).splitlines()[-1])) sys.stdout.flush() else: @@ -6675,7 +6685,8 @@ def anon_handle_download_build(options, session, args): # We want the latest build, not a specific build try: builds = session.listTagged(suboptions.latestfrom, latest=True, package=build, type=suboptions.type) - except koji.GenericError, data: + except koji.GenericError: + data = sys.exc_info()[1] print("Error finding latest build: %s" % data) return 1 if not builds: @@ -7322,7 +7333,8 @@ def handle_runroot(options, session, args): kwargs['new_chroot'] = True task_id = session.runroot(tag, arch, command, **kwargs) - except koji.GenericError, e: + except koji.GenericError: + e = sys.exc_info()[1] if 'Invalid method' in str(e): print("* The runroot plugin appears to not be installed on the" " koji hub. Please contact the administrator.") @@ -7398,8 +7410,8 @@ def handle_save_failed_tree(options, session, args): try: task_id = session.saveFailedTree(br_id, opts.full) - except koji.GenericError as e: - m = str(e) + except koji.GenericError: + m = str(sys.exc_info()[1]) if 'Invalid method' in m: print(_("* The save_failed_tree plugin appears to not be " "installed on the koji hub. Please contact the " @@ -7513,9 +7525,11 @@ def activate_session(session): session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) else: session.krb_login(proxyuser=options.runas) - except socket.error, e: + except socket.error: + e = sys.exc_info()[1] warn(_("Could not connect to Kerberos authentication service: %s") % e.args[1]) - except Exception, e: + except Exception: + e = sys.exc_info()[1] if krbV is not None and isinstance(e, krbV.Krb5Error): error(_("Kerberos authentication failed: %s (%s)") % (e.args[1], e.args[0])) else: diff --git a/koji/__init__.py b/koji/__init__.py index 837d504..f09b8ca 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -493,8 +493,8 @@ def safe_xmlrpc_loads(s): """Load xmlrpc data from a string, but catch faults""" try: return loads(s) - except Fault, f: - return f + except Fault: + return sys.exc_info()[1] ## BEGIN kojikamid dup @@ -1573,7 +1573,8 @@ def config_directory_contents(dir_name): configs = [] try: conf_dir_contents = os.listdir(dir_name) - except OSError, exception: + except OSError: + exception = sys.exc_info()[1] if exception.errno != errno.ENOENT: raise else: @@ -2304,7 +2305,8 @@ class ClientSession(object): for i in (0, 1): try: return self._sendOneCall(handler, headers, request) - except Exception, e: + except Exception: + e = sys.exc_info()[1] if i or not is_conn_error(e): raise self.logger.debug("Connection Error: %s", e) @@ -2397,7 +2399,8 @@ class ClientSession(object): # note that, for logged-in sessions the server should tell us (via a RetryError fault) # if the call cannot be retried. For non-logged-in sessions, all calls should be read-only # and hence retryable. - except Fault, fault: + except Fault: + fault = sys.exc_info()[1] #try to convert the fault to a known exception err = convertFault(fault) if isinstance(err, ServerOffline): @@ -2413,7 +2416,8 @@ class ClientSession(object): except (SystemExit, KeyboardInterrupt): #(depending on the python version, these may or may not be subclasses of Exception) raise - except Exception, e: + except Exception: + e = sys.exc_info()[1] tb_str = ''.join(traceback.format_exception(*sys.exc_info())) self.new_session() @@ -2937,7 +2941,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15'): # decode it using the fallback encoding. try: return value.decode('utf8').encode('utf8') - except UnicodeDecodeError, err: + except UnicodeDecodeError: return value.decode(fallback).encode('utf8') else: return value diff --git a/koji/util.py b/koji/util.py index a7213df..6156ae2 100644 --- a/koji/util.py +++ b/koji/util.py @@ -157,12 +157,13 @@ def call_with_argcheck(func, args, kwargs=None): kwargs = {} try: return func(*args, **kwargs) - except TypeError, e: - if sys.exc_info()[2].tb_next is None: + except TypeError: + ei = sys.exc_info() + if ei[2].tb_next is None: # The stack is only one high, so the error occurred in this function. # Therefore, we assume the TypeError is due to a parameter mismatch # in the above function call. - raise koji.ParameterError(str(e)) + raise koji.ParameterError(str(ei[1])) raise @@ -460,7 +461,8 @@ def setup_rlimits(opts, logger=None): logger.warn('Setting resource limit: %s = %r', key, limits) try: resource.setrlimit(rcode, tuple(limits)) - except ValueError, e: + except ValueError: + e = sys.exc_info()[1] logger.error("Unable to set %s: %s", key, e) class adler32_constructor(object): From 87e15a77401a8f14d23deb9ff1c3c3147f4ecf3b Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 4/11] unicode handling --- diff --git a/cli/koji b/cli/koji index e90a455..5a12fc5 100755 --- a/cli/koji +++ b/cli/koji @@ -110,6 +110,13 @@ greetings = ('hello', 'hi', 'yo', "what's up", "g'day", 'back to work', u'नमस्कार', u'안녕하세요') +def printable_unicode(s): + # python3 unicode handling + if sys.version_info[0] < 3: + return s.encode('utf-8') + else: + return s + def _(args): """Stub function for translation""" return args @@ -3994,15 +4001,15 @@ def _printInheritance(tags, sibdepths=None, reverse=False): if depth < currtag['currdepth']: outspacing = depth - outdepth sys.stdout.write(' ' * (outspacing * 3 - 1)) - sys.stdout.write(u'\u2502'.encode('UTF-8')) + sys.stdout.write(printable_unicode(u'\u2502')) outdepth = depth sys.stdout.write(' ' * ((currtag['currdepth'] - outdepth) * 3 - 1)) if siblings: - sys.stdout.write(u'\u251c'.encode('UTF-8')) + sys.stdout.write(printable_unicode(u'\u251c')) else: - sys.stdout.write(u'\u2514'.encode('UTF-8')) - sys.stdout.write(u'\u2500'.encode('UTF-8')) + sys.stdout.write(printable_unicode(u'\u2514')) + sys.stdout.write(printable_unicode(u'\u2500')) if reverse: sys.stdout.write('%(name)s (%(tag_id)i)\n' % currtag) else: @@ -7275,7 +7282,7 @@ def handle_moshimoshi(options, session, args): if not u: print("Not authenticated") u = {'name' : 'anonymous user'} - print("%s, %s!" % (random.choice(greetings).encode('utf-8'), u["name"])) + print("%s, %s!" % (printable_unicode(random.choice(greetings)), u["name"])) print("") print("You are using the hub at %s" % session.baseurl) authtype = u.get('authtype', getattr(session, 'authtype', None)) From dab5fe83b8ebfe62899e54b6ac85a98a6f5e5981 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 5/11] octal number --- diff --git a/cli/koji b/cli/koji index 5a12fc5..cab8121 100755 --- a/cli/koji +++ b/cli/koji @@ -1585,7 +1585,7 @@ def handle_restart_hosts(options, session, args): def linked_upload(localfile, path, name=None): """Link a file into the (locally writable) workdir, bypassing upload""" - old_umask = os.umask(002) + old_umask = os.umask(2) # o002 try: if name is None: name = os.path.basename(localfile) From eb5d22e93c96a1701b97b4815738b376551b6f50 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 6/11] fail correctly on missing krbV --- diff --git a/koji/__init__.py b/koji/__init__.py index f09b8ca..01d5eed 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -25,6 +25,7 @@ import sys try: import krbV except ImportError: # pragma: no cover + krbV = None sys.stderr.write("Warning: Could not install krbV module. Kerberos support will be disabled.\n") sys.stderr.flush() import base64 From f30787e9ff7ff343532de25984ab26a0b9ac1a37 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 7/11] fix test --- diff --git a/tests/test_utils.py b/tests/test_utils.py index 88cd1aa..68d253e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -81,7 +81,7 @@ class MiscFunctionTestCase(unittest.TestCase): islink.assert_called_once_with(dst) move.assert_not_called() - @mock.patch('urllib2.urlopen') + @mock.patch('koji.urllib_urlopen') @mock.patch('tempfile.TemporaryFile') @mock.patch('shutil.copyfileobj') @mock.patch('__builtin__.open') From 63f0a374f580b2ca51ba86415ddbc8031dfa0cbe Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 8/11] remove long int literal --- diff --git a/koji/util.py b/koji/util.py index 6156ae2..2c060e8 100644 --- a/koji/util.py +++ b/koji/util.py @@ -469,12 +469,12 @@ class adler32_constructor(object): #mimicing the hashlib constructors def __init__(self, arg=''): - self._value = adler32(arg) & 0xffffffffL + self._value = adler32(arg) & 0xffffffff #the bitwise and works around a bug in some versions of python #see: https://bugs.python.org/issue1202 def update(self, arg): - self._value = adler32(arg, self._value) & 0xffffffffL + self._value = adler32(arg, self._value) & 0xffffffff def digest(self): return self._value From fedd5fbc535e225ce27a3dd41364895124f06a77 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 10 2017 16:53:28 +0000 Subject: [PATCH 9/11] replace urlgrabber with pycurl --- diff --git a/cli/koji b/cli/koji index cab8121..c502f92 100755 --- a/cli/koji +++ b/cli/koji @@ -53,14 +53,13 @@ import logging import os import re import pprint +import pycurl import random import socket import stat import string import time import traceback -import urlgrabber as grabber -import urlgrabber.progress as progress try: import xmlrpclib except ImportError: @@ -6774,15 +6773,29 @@ def anon_handle_download_build(options, session, args): url = pathinfo.build(info) + '/' + fname urls.append((url, os.path.basename(fname))) - if suboptions.quiet: - pg = None - else: - pg = progress.TextMeter() + def _progress(download_t, download_d, upload_t, upload_d): + if download_t == 0: + percent_done = 0.0 + else: + percent_done = float(download_d)/float(download_t) + percent_done_str = "%02d%%" % (percent_done * 100) + data_done = _format_size(download_d) + + sys.stdout.write("[% -36s] % 4s % 10s\r" % ('='*(int(percent_done * 36)), percent_done_str, data_done)) + sys.stdout.flush() for url, relpath in urls: if '/' in relpath: koji.ensuredir(os.path.dirname(relpath)) - grabber.urlgrab(url, filename=relpath, progress_obj=pg, text=relpath) + print(relpath) + c = pycurl.Curl() + c.setopt(c.URL, url) + c.setopt(c.WRITEDATA, open(relpath, 'wb')) + if not suboptions.quiet: + c.setopt(c.NOPROGRESS, False) + c.setopt(c.XFERINFOFUNCTION, _progress) + c.perform() + print('') def anon_handle_download_logs(options, session, args): diff --git a/docs/source/writing_koji_code.rst b/docs/source/writing_koji_code.rst index 2d92dea..fd6f4f1 100644 --- a/docs/source/writing_koji_code.rst +++ b/docs/source/writing_koji_code.rst @@ -667,7 +667,7 @@ You will need to install the following packages to actually run the tests. * ``python-krbV`` * ``python-mock`` * ``python-simplejson`` - * ``python-urlgrabber`` + * ``python-pycurl`` * ``python-psycopg2`` * ``python-requests`` * ``python-qpid-proton`` diff --git a/koji.spec b/koji.spec index 250c1a2..dc7c2bb 100644 --- a/koji.spec +++ b/koji.spec @@ -31,6 +31,7 @@ Requires: pyOpenSSL Requires: python-requests Requires: python-requests-kerberos Requires: python-urlgrabber +Requires: python-pycurl Requires: python-dateutil BuildRequires: python %if %{use_systemd} From 68756def794585dcd763f8b54caf168388334928 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 11 2017 07:10:25 +0000 Subject: [PATCH 10/11] remove urlgrabber completely --- diff --git a/koji.spec b/koji.spec index dc7c2bb..f104c2e 100644 --- a/koji.spec +++ b/koji.spec @@ -30,7 +30,6 @@ Requires: rpm-python Requires: pyOpenSSL Requires: python-requests Requires: python-requests-kerberos -Requires: python-urlgrabber Requires: python-pycurl Requires: python-dateutil BuildRequires: python diff --git a/util/koji-shadow b/util/koji-shadow index 233d33d..b6e9fed 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -39,7 +39,6 @@ import string import sys import time import urllib2 -import urlgrabber.grabber as grabber import xmlrpclib # for ProtocolError and Fault import rpm diff --git a/vm/kojivmd b/vm/kojivmd index b003b54..32d014f 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -40,8 +40,8 @@ import SimpleXMLRPCServer import threading import base64 import pwd -import urlgrabber import fnmatch +import pycurl from ConfigParser import ConfigParser from optparse import OptionParser try: @@ -665,7 +665,10 @@ class VMExecTask(BaseTaskHandler): else: raise koji.BuildError('unsupported file type: %s' % type) koji.ensuredir(os.path.dirname(localpath)) - urlgrabber.urlgrab(remote_url, filename=localpath) + c = pycurl.Curl() + c.setopt(c.URL, remote_url) + c.setopt(c.WRITEDATA, open(localpath, 'wb')) + c.perform() return file(localpath, 'r') From bf313d9ab01067f4724cdd58d55ba6403f3ab0bf Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 11 2017 07:33:32 +0000 Subject: [PATCH 11/11] group python3 imports --- diff --git a/cli/koji b/cli/koji index c502f92..f006c72 100755 --- a/cli/koji +++ b/cli/koji @@ -40,16 +40,13 @@ except ImportError: # pragma: no cover import simplejson as json except ImportError: json = None -try: - import ConfigParser -except ImportError: # pragma: no cover - import configparser as ConfigParser import base64 import dateutil.parser import koji import koji.util import fnmatch import logging +import optparse import os import re import pprint @@ -61,10 +58,6 @@ import string import time import traceback try: - import xmlrpclib -except ImportError: - import xmlrpc.client as xmlrpclib -try: import libcomps except ImportError: # pragma: no cover libcomps = None @@ -72,7 +65,12 @@ except ImportError: # pragma: no cover import yum.comps as yumcomps except ImportError: yumcomps = None -import optparse +if sys.version_info[0] < 3: + import ConfigParser + import xmlrpclib +else: # pragma: no cover + import configparser as ConfigParser + import xmlrpc.client as xmlrpclib # fix OptionParser for python 2.3 (optparse verion 1.4.1+) @@ -113,7 +111,7 @@ def printable_unicode(s): # python3 unicode handling if sys.version_info[0] < 3: return s.encode('utf-8') - else: + else: # pragma: no cover return s def _(args): diff --git a/koji/__init__.py b/koji/__init__.py index 01d5eed..67d3122 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -30,16 +30,8 @@ except ImportError: # pragma: no cover sys.stderr.flush() import base64 import datetime -try: - import ConfigParser -except ImportError: # pragma: no cover - import configparser as ConfigParser import errno from fnmatch import fnmatch -try: - import httplib -except ImportError: - import http.client as httplib import imp import logging import logging.handlers @@ -76,30 +68,31 @@ import tempfile import time import traceback import urllib -try: +import warnings +import xml.sax +import xml.sax.handler +if sys.version_info[0] < 3: + import ConfigParser + import httplib import urllib2 import urlparse urllib_urlopen = urllib2.urlopen urllib_splitport = urllib.splitport urllib_urlencode = urllib.urlencode urlparse_urlsplit = urlparse.urlsplit -except ImportError: + import xmlrpclib + from xmlrpclib import loads, dumps, Fault +else: # pragma: no cover + import configparser as ConfigParser + import http.client as httplib urllib_urlopen = urllib.request.urlopen urllib_splitport = urllib.parse.splitport urllib_urlencode = urllib.parse.urlencode urlparse_urlsplit = urllib.parse.urlsplit -import warnings -import xml.sax -import xml.sax.handler -try: - import xmlrpclib - from xmlrpclib import loads, dumps, Fault -except: # pragma: no cover import xmlrpc.client as xmlrpclib loads = xmlrpclib.loads dumps = xmlrpclib.dumps Fault = xmlrpclib.Fault - from koji.util import md5_constructor, adler32_constructor PROFILE_MODULES = {} # {module_name: module_instance} diff --git a/koji/util.py b/koji/util.py index 2c060e8..41e2b5c 100644 --- a/koji/util.py +++ b/koji/util.py @@ -30,9 +30,9 @@ import shutil import stat import sys import time -try: +if sys.version_info[0] < 3: import ConfigParser -except ImportError: # pragma: no cover +else: # pragma: no cover import configparser as ConfigParser from zlib import adler32