From 929fd4c1f158eee178c4c2cc4f2b27b8b9c3c15f Mon Sep 17 00:00:00 2001 From: Xibo Ning Date: Nov 24 2016 06:58:39 +0000 Subject: [PATCH 1/6] add method listFlatPackages and CountAndFilterResults --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 51e41a2..2a36fa7 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9842,7 +9842,34 @@ class RootExports(object): return results - def checkTagPackage(self, tag, pkg): + + def listFlatPackages(self, prefix=None, queryOpts=None): + """list packages that starts with prefix and are filted + and ordered by queryOpts. + + Args: + prefix: default is None. If is not None will filter out + packages which name doesn't start with the prefix. + queryOpts: query options used by the QueryProcessor. + + Returns: + A list of maps is returned, and each map contains key + 'package_name' and 'package_id'. + """ + _escape = lambda _str: _str.replace('_', '#_').replace('%', '#%') + + if prefix is None: + clauses = None + else: + clauses = ["""package.name ILIKE '%s%%' ESCAPE '#'""" + % _escape(prefix)] + query = QueryProcessor( + tables=['package'], clauses=clauses, + columns=['package.id', 'package.name'], opts=queryOpts) + return query.executeOne() + + + def checkTagPackage(self,tag,pkg): """Check that pkg is in the list for tag. Returns true/false""" tag_id = get_tag_id(tag, strict=False) pkg_id = get_package_id(pkg, strict=False) @@ -10538,6 +10565,56 @@ class RootExports(object): return results + + def CountAndFilterResults(self, methodName, *args, **kw): + """ Replacement of the method filterResults when we need both the total + result count and the filtered result. + + Execute the XML-RPC method with the given name and filter the results + based on the options specified in the keywork option "filterOpts". + The method must return a list of maps. Any other return type will + result in a TypeError. + + Args: + offset: the number of elements to trim off the front of the list + limit: the maximum number of results to return + order: the map key to use to sort the list; the list will be sorted + before offset or limit are applied + noneGreatest: when sorting, consider 'None' to be greater than all + other values; python considers None less than all other values, + but Postgres sorts NULL higher than all other values; default + to True for consistency with database sorts + + Returns: + Tuple of total result amount and the filtered result. + """ + filterOpts = kw.pop('filterOpts', {}) + + results = getattr(self, methodName)(*args, **kw) + if result is None: + return 0, None + elif isinstance(result, list): + _count = len(result) + else: + _count = 1 + + if not isinstance(results, list): + raise TypeError, '%s() did not return a list' % methodName + + order = filterOpts.get('order') + if order: + results.sort(self._sortByKeyFunc(order, filterOpts.get('noneGreatest', True))) + + offset = filterOpts.get('offset') + if offset is not None: + results = results[offset:] + limit = filterOpts.get('limit') + if limit is not None: + results = results[:limit] + + return _count, result + + def getBuildNotifications(self, userID=None): """Get build notifications for the user with the given ID. If no ID is specified, get the notifications for the currently logged-in user. If From 5fe8824f191e280d02290622aaaef9ba205c7779 Mon Sep 17 00:00:00 2001 From: Xibo Ning Date: Nov 24 2016 07:00:43 +0000 Subject: [PATCH 2/6] fix method names to listPackagesSimple and countAndFilterResults. 1. replace listPackages call by listPackagesSimple. 2. replace filterResults by countAndFilterResults. --- diff --git a/cli/koji b/cli/koji index 202175c..9c26a2e 100755 --- a/cli/koji +++ b/cli/koji @@ -3216,7 +3216,10 @@ def anon_handle_list_pkgs(options, session, args): opts['event'] = event['id'] event['timestr'] = time.asctime(time.localtime(event['ts'])) print "Querying at event %(id)i (%(timestr)s)" % event - data = session.listPackages(**opts) + if 'tagID' in opts or 'pkgID' in opts or 'userID' in opts: + data = session.listPackages(**opts) + else: + data = session.listPackagesSimple(**opts) if not data: print "(no matching packages)" return 1 @@ -6196,7 +6199,10 @@ def handle_set_pkg_owner_global(options, session, args): continue to_change.extend(entries) if not packages and options.old_user: - entries = session.listPackages(**opts) + if 'tagID' in opts or 'pkgID' in opts or 'userID' in opts: + entries = session.listPackages(**opts) + else: + entries = session.listPackagesSimple(**opts) if not entries: print "No data for user %s" % old_user['name'] return 1 diff --git a/hub/kojihub.py b/hub/kojihub.py index 2a36fa7..a3f5fb6 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9843,7 +9843,7 @@ class RootExports(object): return results - def listFlatPackages(self, prefix=None, queryOpts=None): + def listPackagesSimple(self, prefix=None, queryOpts=None): """list packages that starts with prefix and are filted and ordered by queryOpts. @@ -9857,16 +9857,18 @@ class RootExports(object): 'package_name' and 'package_id'. """ _escape = lambda _str: _str.replace('_', '#_').replace('%', '#%') - + fields = (('package.id', 'package_id'), + ('package.name', 'package_name')) if prefix is None: clauses = None else: - clauses = ["""package.name ILIKE '%s%%' ESCAPE '#'""" - % _escape(prefix)] + prefix = _escape(prefix) + clauses = ["""package.name ILIKE '%(prefix)s%%' ESCAPE '#'"""] query = QueryProcessor( - tables=['package'], clauses=clauses, - columns=['package.id', 'package.name'], opts=queryOpts) - return query.executeOne() + tables=['package'], clauses=clauses, values=locals(), + columns=[f[0] for f in fields], aliases=[f[1] for f in fields], + opts=queryOpts) + return query.execute() def checkTagPackage(self,tag,pkg): @@ -10544,31 +10546,11 @@ class RootExports(object): NULL higher than all other values; default to True for consistency with database sorts """ - filterOpts = kw.pop('filterOpts', {}) - - results = getattr(self, methodName)(*args, **kw) - if results is None: - return None - elif not isinstance(results, list): - raise TypeError, '%s() did not return a list' % methodName - - order = filterOpts.get('order') - if order: - results.sort(self._sortByKeyFunc(order, filterOpts.get('noneGreatest', True))) - - offset = filterOpts.get('offset') - if offset is not None: - results = results[offset:] - limit = filterOpts.get('limit') - if limit is not None: - results = results[:limit] - - return results + return self.countAndFilterResults(methodName, *args, **kw)[1] - def CountAndFilterResults(self, methodName, *args, **kw): - """ Replacement of the method filterResults when we need both the total - result count and the filtered result. + def countAndFilterResults(self, methodName, *args, **kw): + """Filter results by a given name and count total result account. Execute the XML-RPC method with the given name and filter the results based on the options specified in the keywork option "filterOpts". diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 0ac8902..c206d99 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -352,8 +352,7 @@ def notificationedit(environ, notificationID): values = _initValues(environ, 'Edit Notification') values['notif'] = notification - packages = server.listPackages() - packages.sort(kojiweb.util.sortByKeyFunc('package_name')) + packages = server.listPackagesSimple(queryOpts={'order': 'package_name'}) values['packages'] = packages tags = server.listTags(queryOpts={'order': 'name'}) values['tags'] = tags @@ -397,8 +396,7 @@ def notificationcreate(environ): values = _initValues(environ, 'Edit Notification') values['notif'] = None - packages = server.listPackages() - packages.sort(kojiweb.util.sortByKeyFunc('package_name')) + packages = server.listPackagesSimple(queryOpts={'order': 'package_name'}) values['packages'] = packages tags = server.listTags(queryOpts={'order': 'name'}) values['tags'] = tags diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index 5aa1cb3..c3784fa 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -330,12 +330,11 @@ def paginateResults(server, values, methodName, args=None, kw=None, if not dataName: raise StandardError, 'dataName must be specified' - totalRows = server.count(methodName, *args, **kw) - kw['filterOpts'] = {'order': order, 'offset': start, 'limit': pageSize} - data = server.filterResults(methodName, *args, **kw) + + totalRows, data = server.countAndFilterResults(methodName, *args, **kw) count = len(data) _populateValues(values, dataName, prefix, data, totalRows, start, count, pageSize, order) From 4698e1964aa65e63bfffc4d71630dc4513ea5e9f Mon Sep 17 00:00:00 2001 From: Xibo Ning Date: Nov 24 2016 07:50:50 +0000 Subject: [PATCH 3/6] have listPackages() method to support queryOpts option and convert the packages page to use paginateMethod() --- diff --git a/cli/koji b/cli/koji index 9c26a2e..202175c 100755 --- a/cli/koji +++ b/cli/koji @@ -3216,10 +3216,7 @@ def anon_handle_list_pkgs(options, session, args): opts['event'] = event['id'] event['timestr'] = time.asctime(time.localtime(event['ts'])) print "Querying at event %(id)i (%(timestr)s)" % event - if 'tagID' in opts or 'pkgID' in opts or 'userID' in opts: - data = session.listPackages(**opts) - else: - data = session.listPackagesSimple(**opts) + data = session.listPackages(**opts) if not data: print "(no matching packages)" return 1 @@ -6199,10 +6196,7 @@ def handle_set_pkg_owner_global(options, session, args): continue to_change.extend(entries) if not packages and options.old_user: - if 'tagID' in opts or 'pkgID' in opts or 'userID' in opts: - entries = session.listPackages(**opts) - else: - entries = session.listPackagesSimple(**opts) + entries = session.listPackages(**opts) if not entries: print "No data for user %s" % old_user['name'] return 1 diff --git a/hub/kojihub.py b/hub/kojihub.py index a3f5fb6..261e34f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9793,7 +9793,7 @@ class RootExports(object): getPackage = staticmethod(lookup_package) - def listPackages(self, tagID=None, userID=None, pkgID=None, prefix=None, inherited=False, with_dups=False, event=None): + def listPackages(self, tagID=None, userID=None, pkgID=None, prefix=None, inherited=False, with_dups=False, event=None, queryOpts=None): """List if tagID and/or userID is specified, limit the list to packages belonging to the given user or with the given tag. @@ -9815,8 +9815,7 @@ class RootExports(object): - blocked """ if tagID is None and userID is None and pkgID is None: - query = """SELECT id, name from package""" - results = _multiRow(query, {}, ('package_id', 'package_name')) + return self.listPackagesSimple(prefix, queryOpts) else: if tagID is not None: tagID = get_tag_id(tagID, strict=True) @@ -9840,7 +9839,7 @@ class RootExports(object): prefix = prefix.lower() results = [package for package in results if package['package_name'].lower().startswith(prefix)] - return results + return _applyQueryOpts(results, queryOpts) def listPackagesSimple(self, prefix=None, queryOpts=None): diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index c206d99..23cdab4 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -828,9 +828,9 @@ def packages(environ, tagID=None, userID=None, order='package_name', start=None, inherited = int(inherited) values['inherited'] = inherited - kojiweb.util.paginateResults(server, values, 'listPackages', - kw={'tagID': tagID, 'userID': userID, 'prefix': prefix, 'inherited': bool(inherited)}, - start=start, dataName='packages', prefix='package', order=order) + packages = kojiweb.util.paginateMethod(server, values, 'listPackages', + kw={'tagID': tagID, 'userID': userID, 'prefix': prefix, 'inherited': bool(inherited)}, + start=start, dataName='packages', prefix='package', order=order) values['chars'] = _PREFIX_CHARS From fb679fa627d3c14a6c83002f066b1850fa726ba7 Mon Sep 17 00:00:00 2001 From: Xibo Ning Date: Nov 24 2016 07:51:57 +0000 Subject: [PATCH 4/6] s/result/results/ --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 261e34f..dff529d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -10549,7 +10549,7 @@ class RootExports(object): def countAndFilterResults(self, methodName, *args, **kw): - """Filter results by a given name and count total result account. + """Filter results by a given name and count total results account. Execute the XML-RPC method with the given name and filter the results based on the options specified in the keywork option "filterOpts". @@ -10567,15 +10567,15 @@ class RootExports(object): to True for consistency with database sorts Returns: - Tuple of total result amount and the filtered result. + Tuple of total results amount and the filtered results. """ filterOpts = kw.pop('filterOpts', {}) results = getattr(self, methodName)(*args, **kw) - if result is None: + if results is None: return 0, None - elif isinstance(result, list): - _count = len(result) + elif isinstance(results, list): + _count = len(results) else: _count = 1 @@ -10593,7 +10593,7 @@ class RootExports(object): if limit is not None: results = results[:limit] - return _count, result + return _count, results def getBuildNotifications(self, userID=None): From 6c570e879e9346532d4ba37bcf2ee22dbb5bafc1 Mon Sep 17 00:00:00 2001 From: Xibo Ning Date: Nov 24 2016 07:59:03 +0000 Subject: [PATCH 5/6] remove useless return value --- diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 23cdab4..cca8785 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -828,9 +828,9 @@ def packages(environ, tagID=None, userID=None, order='package_name', start=None, inherited = int(inherited) values['inherited'] = inherited - packages = kojiweb.util.paginateMethod(server, values, 'listPackages', - kw={'tagID': tagID, 'userID': userID, 'prefix': prefix, 'inherited': bool(inherited)}, - start=start, dataName='packages', prefix='package', order=order) + kojiweb.util.paginateMethod(server, values, 'listPackages', + kw={'tagID': tagID, 'userID': userID, 'prefix': prefix, 'inherited': bool(inherited)}, + start=start, dataName='packages', prefix='package', order=order) values['chars'] = _PREFIX_CHARS From ec7d124f68826fd678b603b0e02eec451aa59fc6 Mon Sep 17 00:00:00 2001 From: Xibo Ning Date: Jan 13 2017 06:51:57 +0000 Subject: [PATCH 6/6] no need to escape, thanks mikem --- diff --git a/hub/kojihub.py b/hub/kojihub.py index dff529d..3d911c8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9855,14 +9855,12 @@ class RootExports(object): A list of maps is returned, and each map contains key 'package_name' and 'package_id'. """ - _escape = lambda _str: _str.replace('_', '#_').replace('%', '#%') fields = (('package.id', 'package_id'), ('package.name', 'package_name')) if prefix is None: clauses = None else: - prefix = _escape(prefix) - clauses = ["""package.name ILIKE '%(prefix)s%%' ESCAPE '#'"""] + clauses = ["""package.name ILIKE %(prefix)s || '%%'"""] query = QueryProcessor( tables=['package'], clauses=clauses, values=locals(), columns=[f[0] for f in fields], aliases=[f[1] for f in fields],