From 3464adae2e8a42f5f1a4249d7478cbb0ffb418a3 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Dec 06 2017 05:32:29 +0000 Subject: [PATCH 1/4] cli: fix changelog encode for PY3 relates [issue#577](https://pagure.io/koji/issue/577) --- diff --git a/koji/__init__.py b/koji/__init__.py index b462dd1..5de0225 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2966,6 +2966,16 @@ def removeNonprintable(value): # expects raw-encoded string, not unicode return value.translate(None, NONPRINTABLE_CHARS) +def fixPrint(value): + if not value: + return str('') + elif six.PY2 and isinstance(value, six.text_type): + return value.encode('utf8') + elif six.PY3 and isinstance(value, six.binary_type): + return value.decode('utf8') + else: + return value + def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False): """ Convert value to a 'str' object encoded as UTF-8. @@ -2976,8 +2986,8 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False): return six.b('') if isinstance(value, six.text_type): - # value is already unicode, so just convert it - # to a utf8-encoded str + # value is already unicode(py3: str), so just convert it + # to a utf8-encoded str(py3: bytes) s = value.encode('utf8') else: # value is a str, but may be encoded in utf8 or some diff --git a/koji/util.py b/koji/util.py index 545e7aa..aa8a44a 100644 --- a/koji/util.py +++ b/koji/util.py @@ -48,9 +48,11 @@ try: except ImportError: # pragma: no cover from sha import new as sha1_constructor + def _changelogDate(cldate): return time.strftime('%a %b %d %Y', time.strptime(koji.formatTime(cldate), '%Y-%m-%d %H:%M:%S')) + def formatChangelog(entries): """Format a list of changelog entries (dicts) into a string representation.""" @@ -59,9 +61,9 @@ def formatChangelog(entries): result += """* %s %s %s -""" % (_changelogDate(entry['date']), entry['author'].encode("utf-8"), - entry['text'].encode("utf-8")) - +""" % (_changelogDate(entry['date']), + koji.fixPrint(entry['author']), + koji.fixPrint(entry['text'])) return result DATE_RE = re.compile(r'(\d+)-(\d+)-(\d+)') diff --git a/tests/test_lib/test_fixEncoding.py b/tests/test_lib/test_fixEncoding.py index 4072724..b5b6f19 100644 --- a/tests/test_lib/test_fixEncoding.py +++ b/tests/test_lib/test_fixEncoding.py @@ -7,6 +7,7 @@ from __future__ import absolute_import import koji import six import unittest +import mock class FixEncodingTestCase(unittest.TestCase): """Main test case container""" @@ -44,6 +45,23 @@ class FixEncodingTestCase(unittest.TestCase): d = a[:-3] + u'\x00\x01' + a[-3:] self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b) + @mock.patch('sys.stdout', new_callable=six.StringIO) + def test_fixPrint(self, stdout): + """Test the fixPrint function""" + expected = '' + for a, b in self.simple_values: + if six.PY3: + self.assertEqual(koji.fixPrint(b), a) + else: + self.assertEqual(koji.fixPrint(b), b) + print(koji.fixPrint(b)) + if six.PY3: + expected = expected + a + '\n' + else: + expected = expected + b + '\n' + actual = stdout.getvalue() + self.assertEqual(actual, expected) + complex_values = [ # [ value, fixed ] [{}, {}], From 1da06800cfbede63d9f2973c1d30e064067538bd Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Dec 06 2017 05:32:51 +0000 Subject: [PATCH 2/4] adjustments to test_formatChangelog --- diff --git a/tests/test_lib/test_utils.py b/tests/test_lib/test_utils.py index fa94afe..8ea53f5 100644 --- a/tests/test_lib/test_utils.py +++ b/tests/test_lib/test_utils.py @@ -1,3 +1,4 @@ +# coding=utf-8 from __future__ import absolute_import import mock import unittest @@ -494,12 +495,39 @@ class MavenUtilTestCase(unittest.TestCase): def test_formatChangelog(self): """Test formatChangelog function""" - entries = {'date': datetime(2017, 10, 10, 12, 34, 56), - 'author': 'koji ', - 'text': 'This is a test release'} - sample = "* Tue Oct 10 2017 {0}\n{1}\n\n".format( - entries['author'].encode('utf-8'), entries['text'].encode('utf-8')) - self.assertEqual(sample, koji.util.formatChangelog((entries,))) + data = [ + { + 'author': 'Happy Koji User - 1.1-1', + 'date': '2017-10-25 08:00:00', + 'date_ts': 1508932800, + 'text': '- Line 1\n- Line 2', + }, + { + 'author': u'Happy \u0138\u014dji \u016cs\u0259\u0155 ', + 'date': '2017-08-28 08:00:00', + 'date_ts': 1503921600, + 'text': '- some changelog entry', + }, + { + 'author': 'Koji Admin - 1.49-6', + 'date': datetime(2017, 10, 10, 12, 34, 56), + 'text': '- mass rebuild', + } + ] + expect = ( +u'''* Wed Oct 25 2017 Happy Koji User - 1.1-1 +- Line 1 +- Line 2 + +* Mon Aug 28 2017 Happy ĸōji Ŭsəŕ +- some changelog entry + +* Tue Oct 10 2017 Koji Admin - 1.49-6 +- mass rebuild + +''') + result = koji.util.formatChangelog(data) + self.assertMultiLineEqual(expect, result) def test_parseTime(self): """Test parseTime function""" From 8a4c75ee7f4ab57b91c63735b12bd4a4427505ac Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Dec 06 2017 05:32:51 +0000 Subject: [PATCH 3/4] rename fixPrint->_fix_print and add a docstring --- diff --git a/koji/__init__.py b/koji/__init__.py index 5de0225..67819e9 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2966,16 +2966,21 @@ def removeNonprintable(value): # expects raw-encoded string, not unicode return value.translate(None, NONPRINTABLE_CHARS) -def fixPrint(value): - if not value: - return str('') - elif six.PY2 and isinstance(value, six.text_type): + +def _fix_print(value): + """Fix a string so it is suitable to print + + In python2, this means we return a utf8 encoded str + In python3, this means we return unicode + """ + if six.PY2 and isinstance(value, six.text_type): return value.encode('utf8') elif six.PY3 and isinstance(value, six.binary_type): return value.decode('utf8') else: return value + def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False): """ Convert value to a 'str' object encoded as UTF-8. diff --git a/koji/util.py b/koji/util.py index aa8a44a..f7f50b5 100644 --- a/koji/util.py +++ b/koji/util.py @@ -62,8 +62,8 @@ def formatChangelog(entries): %s """ % (_changelogDate(entry['date']), - koji.fixPrint(entry['author']), - koji.fixPrint(entry['text'])) + koji._fix_print(entry['author']), + koji._fix_print(entry['text'])) return result DATE_RE = re.compile(r'(\d+)-(\d+)-(\d+)') diff --git a/tests/test_lib/test_fixEncoding.py b/tests/test_lib/test_fixEncoding.py index b5b6f19..2f7ca8f 100644 --- a/tests/test_lib/test_fixEncoding.py +++ b/tests/test_lib/test_fixEncoding.py @@ -46,15 +46,15 @@ class FixEncodingTestCase(unittest.TestCase): self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b) @mock.patch('sys.stdout', new_callable=six.StringIO) - def test_fixPrint(self, stdout): - """Test the fixPrint function""" + def test_fix_print(self, stdout): + """Test the _fix_print function""" expected = '' for a, b in self.simple_values: if six.PY3: - self.assertEqual(koji.fixPrint(b), a) + self.assertEqual(koji._fix_print(b), a) else: - self.assertEqual(koji.fixPrint(b), b) - print(koji.fixPrint(b)) + self.assertEqual(koji._fix_print(b), b) + print(koji._fix_print(b)) if six.PY3: expected = expected + a + '\n' else: From 377fecbbfc4a8db8db1865dbd89d38046b6d33a0 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Dec 06 2017 06:08:37 +0000 Subject: [PATCH 4/4] fix unit test - `test_formatChangelog` --- diff --git a/tests/test_lib/test_utils.py b/tests/test_lib/test_utils.py index 8ea53f5..087b52f 100644 --- a/tests/test_lib/test_utils.py +++ b/tests/test_lib/test_utils.py @@ -515,7 +515,7 @@ class MavenUtilTestCase(unittest.TestCase): } ] expect = ( -u'''* Wed Oct 25 2017 Happy Koji User - 1.1-1 +'''* Wed Oct 25 2017 Happy Koji User - 1.1-1 - Line 1 - Line 2