From eeeeb3ab782dd7fc089d7d2ab62026d98057df52 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 04 2019 15:13:51 +0000 Subject: skip fixEncoding for py3 --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 9136bd5..5322a64 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4361,10 +4361,7 @@ def _get_zipfile_list(archive_id, zippath): return result with zipfile.ZipFile(zippath, 'r') as archive: for entry in archive.infolist(): - if six.PY2: - filename = koji.fixEncoding(entry.filename) - else: - filename = entry.filename + filename = koji.fixEncoding(entry.filename) result.append({'archive_id': archive_id, 'name': filename, 'size': entry.file_size, @@ -4389,10 +4386,7 @@ def _get_tarball_list(archive_id, tarpath): return result with tarfile.open(tarpath, 'r') as archive: for entry in archive: - if six.PY2: - filename = koji.fixEncoding(entry.name) - else: - filename = entry.name + filename = koji.fixEncoding(entry.name) result.append({'archive_id': archive_id, 'name': filename, 'size': entry.size, @@ -6368,10 +6362,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No archiveinfo = {'buildroot_id': buildroot_id} archiveinfo['build_id'] = buildinfo['id'] if metadata_only: - if six.PY2: - filename = koji.fixEncoding(fileinfo['filename']) - else: - filename = fileinfo['filename'] + filename = koji.fixEncoding(fileinfo['filename']) archiveinfo['filename'] = filename archiveinfo['size'] = fileinfo['filesize'] archiveinfo['checksum'] = fileinfo['checksum'] @@ -6382,10 +6373,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']] archiveinfo['metadata_only'] = True else: - if six.PY2: - filename = koji.fixEncoding(os.path.basename(filepath)) - else: - filename = os.path.basename(filepath) + filename = koji.fixEncoding(os.path.basename(filepath)) archiveinfo['filename'] = filename archiveinfo['size'] = os.path.getsize(filepath) # trust values computed on hub (CG_Importer.prep_outputs) @@ -6507,8 +6495,7 @@ def _import_archive_file(filepath, destdir): be created. """ fname = os.path.basename(filepath) - if six.PY2: - fname = koji.fixEncoding(fname) + fname = koji.fixEncoding(fname) final_path = "%s/%s" % (destdir, fname) if os.path.exists(final_path): raise koji.GenericError("Error importing archive file, %s already exists" % final_path) @@ -7693,10 +7680,7 @@ def parse_json(value, desc=None, errstr=None): if value is None: return value try: - if six.PY2: - return koji.fixEncodingRecurse(json.loads(value)) - else: - return json.loads(value) + return koji.fixEncodingRecurse(json.loads(value)) except Exception: if errstr is None: if desc is None: @@ -9698,9 +9682,8 @@ class RootExports(object): for (cltime, clname, cltext) in zip(fields['changelogtime'], fields['changelogname'], fields['changelogtext']): cldate = datetime.datetime.fromtimestamp(cltime).isoformat(' ') - if six.PY2: - clname = koji.fixEncoding(clname) - cltext = koji.fixEncoding(cltext) + clname = koji.fixEncoding(clname) + cltext = koji.fixEncoding(cltext) if author and author != clname: continue @@ -10260,8 +10243,7 @@ class RootExports(object): raise koji.GenericError('either rpmID or taskID and filepath must be specified') headers = koji.get_header_fields(rpm_path, headers) - return koji.fixEncodingRecurse(headers, remove_nonprintable=True, - ignore_keys=True) + return koji.fixEncodingRecurse(headers, remove_nonprintable=True) queryRPMSigs = staticmethod(query_rpm_sigs) diff --git a/koji/__init__.py b/koji/__init__.py index de79588..2cdd3ec 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -3063,10 +3063,16 @@ def _taskLabel(taskInfo): return '%s (%s)' % (method, arch) CONTROL_CHARS = [chr(i) for i in range(32)] -NONPRINTABLE_CHARS = six.b(''.join([c for c in CONTROL_CHARS if c not in '\r\n\t'])) +NONPRINTABLE_CHARS = ''.join([c for c in CONTROL_CHARS if c not in '\r\n\t']) +if six.PY3: + NONPRINTABLE_CHARS_TABLE = dict.fromkeys(map(ord, NONPRINTABLE_CHARS), None) def removeNonprintable(value): # expects raw-encoded string, not unicode - return value.translate(None, NONPRINTABLE_CHARS) + if six.PY2: + return value.translate(None, NONPRINTABLE_CHARS) + else: + return value.translate(NONPRINTABLE_CHARS_TABLE) + def _fix_print(value): @@ -3089,6 +3095,12 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False): If value is not valid UTF-8 to begin with, assume it is encoded in the 'fallback' charset. """ + if six.PY3: + if remove_nonprintable: + return removeNonprintable(value) + else: + return value + if not value: return six.b('') @@ -3110,11 +3122,15 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False): return s -def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False, ignore_keys=False): +def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): """Recursively fix string encoding in an object Similar behavior to fixEncoding, but recursive """ + if six.PY3: + # don't bother with fixing in py3 + return value + if isinstance(value, tuple): return tuple([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value]) elif isinstance(value, list): @@ -3123,9 +3139,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False, ret = {} for k in value: v = fixEncodingRecurse(value[k], fallback=fallback, remove_nonprintable=remove_nonprintable) - # expect, that keys are never really binary - if not ignore_keys: - k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable) + k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable) ret[k] = v return ret elif isinstance(value, six.text_type): diff --git a/tests/test_builder/test_build_notification.py b/tests/test_builder/test_build_notification.py index 3abb8a9..0d35f6e 100644 --- a/tests/test_builder/test_build_notification.py +++ b/tests/test_builder/test_build_notification.py @@ -100,5 +100,5 @@ class TestBuildNotification(unittest.TestCase): self.assertEqual(recipients, ["user@example.com"]) fn = os.path.join(os.path.dirname(__file__), 'data/calls', 'build_notif_1', 'message.txt') with open(fn, 'rb') as fp: - msg_expect = fp.read() + msg_expect = fp.read().decode() self.assertEqual(message, msg_expect) diff --git a/tests/test_lib/test_fixEncoding.py b/tests/test_lib/test_fixEncoding.py index 0388a99..e53622e 100644 --- a/tests/test_lib/test_fixEncoding.py +++ b/tests/test_lib/test_fixEncoding.py @@ -18,52 +18,48 @@ class FixEncodingTestCase(unittest.TestCase): simple_values = [ # [ value, fixed ] - ['', six.b('')], - [u'', six.b('')], - [u'góðan daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')], - [u'hej', six.b('hej')], - [u'zdravstvuite', six.b('zdravstvuite')], - [u'céad míle fáilte', six.b('c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte')], - [u'dobrý den', six.b('dobr\xc3\xbd den')], - [u'hylô', six.b('hyl\xc3\xb4')], - [u'jó napot', six.b('j\xc3\xb3 napot')], - [u'tervehdys', six.b('tervehdys')], - [u'olá', six.b('ol\xc3\xa1')], - [u'grüezi', six.b('gr\xc3\xbcezi')], - [u'dobre dan', six.b('dobre dan')], - [u'hello', six.b('hello')], - [u'bună ziua', six.b('bun\xc4\x83 ziua')], - [u'こんにちは', six.b('\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf')], - [u'你好', six.b('\xe4\xbd\xa0\xe5\xa5\xbd')], - [u'नमस्कार', six.b('\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0')], - [u'안녕하세요', six.b('\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94')], + ['', ''], + [u'', ''], + [u'góðan daginn', 'g\xc3\xb3\xc3\xb0an daginn'], + [u'hej', 'hej'], + [u'zdravstvuite', 'zdravstvuite'], + [u'céad míle fáilte', 'c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte'], + [u'dobrý den', 'dobr\xc3\xbd den'], + [u'hylô', 'hyl\xc3\xb4'], + [u'jó napot', 'j\xc3\xb3 napot'], + [u'tervehdys', 'tervehdys'], + [u'olá', 'ol\xc3\xa1'], + [u'grüezi', 'gr\xc3\xbcezi'], + [u'dobre dan', 'dobre dan'], + [u'hello', 'hello'], + [u'bună ziua', 'bun\xc4\x83 ziua'], + [u'こんにちは', '\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf'], + [u'你好', '\xe4\xbd\xa0\xe5\xa5\xbd'], + [u'नमस्कार', '\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0'], + [u'안녕하세요', '\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94'], ] def test_fixEncoding(self): """Test the fixEncoding function""" for a, b in self.simple_values: - self.assertEqual(koji.fixEncoding(a), b) self.assertEqual(koji.fixEncoding(b), b) - c = a.encode('utf16') - self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b) - d = a[:-3] + u'\x00\x01' + a[-3:] - self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b) + if six.PY2: + self.assertEqual(koji.fixEncoding(a), b) + c = a.encode('utf16') + self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b) + d = a[:-3] + u'\x00\x01' + a[-3:] + self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b) + else: + self.assertEqual(koji.fixEncoding(a), a) - @mock.patch('sys.stdout', new_callable=six.StringIO) - def test_fix_print(self, stdout): + def test_fix_print(self): """Test the _fix_print function""" - expected = '' + actual, expected = [], [] for a, b in self.simple_values: - if six.PY3: - self.assertEqual(koji._fix_print(b), a) - else: - self.assertEqual(koji._fix_print(b), b) - print(koji._fix_print(b)) - if six.PY3: - expected = expected + a + '\n' - else: - expected = expected + b + '\n' - actual = stdout.getvalue() + actual.append(koji._fix_print(b)) + expected.append(b) + expected = '\n'.join(expected) + actual = '\n'.join(actual) self.assertEqual(actual, expected) complex_values = [ @@ -73,21 +69,24 @@ class FixEncodingTestCase(unittest.TestCase): [None, None], [[], []], [{u'a': 'a' , 'b' : {'c': u'c\x00'}}, - {six.b('a'): six.b('a') , six.b('b') : {six.b('c'): six.b('c\x00')}}], + {'a': 'a' , 'b' : {'c': 'c\x00'}}], # iso8859-15 fallback - ['g\xf3\xf0an daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')], + ['g\xf3\xf0an daginn', 'g\xc3\xb3\xc3\xb0an daginn'], ] nonprint = [ - ['hello\0world\0', six.b('helloworld')], - [u'hello\0world\0', six.b('helloworld')], - [[u'hello\0world\0'], [six.b('helloworld')]], - [{0: u'hello\0world\0'}, {0: six.b('helloworld')}], - [[{0: u'hello\0world\0'}], [{0: six.b('helloworld')}]], + ['hello\0world\0', 'helloworld'], + [u'hello\0world\0', 'helloworld'], + [[u'hello\0world\0'], ['helloworld']], + [{0: u'hello\0world\0'}, {0: 'helloworld'}], + [[{0: u'hello\0world\0'}], [{0: 'helloworld'}]], ] def test_fixEncodingRecurse(self): """Test the fixEncodingRecurse function""" + if six.PY3: + # don't test for py3 + return for a, b in self.simple_values: self.assertEqual(koji.fixEncoding(a), b) for a, b in self.complex_values: diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 460f7d1..04248dc 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1382,9 +1382,9 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i values['enhances'] = server.getRPMDeps(rpm['id'], koji.DEP_ENHANCE) values['enhances'].sort(key=_sortbyname) headers = server.getRPMHeaders(rpm['id'], headers=['summary', 'description', 'license']) - values['summary'] = koji.fixEncoding(str(headers.get('summary'))) - values['description'] = koji.fixEncoding(str(headers.get('description'))) - values['license'] = koji.fixEncoding(str(headers.get('license'))) + values['summary'] = koji.fixEncoding(headers.get('summary')) + values['description'] = koji.fixEncoding(headers.get('description')) + values['license'] = koji.fixEncoding(headers.get('license')) buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'rpmID': rpm['id']}, start=buildrootStart, dataName='buildroots', prefix='buildroot', order=buildrootOrder) diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index ed4571f..f96b131 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -531,9 +531,6 @@ def escapeHTML(value): return value value = koji.fixEncoding(value) - if six.PY3: - # it is bytes now, so decode to str - value = value.decode() return value.replace('&', '&').\ replace('<', '<').\ replace('>', '>')