| """Tests for TR46 code.""" |
| |
| import os.path |
| import re |
| import unittest |
| |
| import idna |
| |
| _RE_UNICODE = re.compile("\\\\u([0-9a-fA-F]{4})") |
| _RE_SURROGATE = re.compile("[\uD800-\uDBFF][\uDC00-\uDFFF]") |
| _SKIP_TESTS = [ |
| # These are strings that are illegal in IDNA 2008. Older versions of the UTS-46 test suite |
| # had these denoted with the 'NV8' marker but this has been removed, so we need to manually |
| # review exceptions and add them here to skip them as text vectors if they are invalid. |
| '\U000102F7\u3002\u200D', |
| '\U0001D7F5\u9681\u2BEE\uFF0E\u180D\u200C', |
| '9\u9681\u2BEE.\u180D\u200C', |
| '\u00DF\u200C\uAAF6\u18A5.\u22B6\u2D21\u2D16', |
| 'ss\u200C\uAAF6\u18A5.\u22B6\u2D21\u2D16', |
| '\u00DF\u200C\uAAF6\u18A5\uFF0E\u22B6\u2D21\u2D16', |
| 'ss\u200C\uAAF6\u18A5\uFF0E\u22B6\u2D21\u2D16', |
| '\U00010A57\u200D\u3002\u2D09\u2D15', |
| '\U00010A57\u200D\uFF61\u2D09\u2D15', |
| '\U0001D7CF\U0001DA19\u2E16.\u200D', |
| '1\U0001DA19\u2E16.\u200D', |
| '\U0001D7E04\U000E01D7\U0001D23B\uFF0E\u200D\U000102F5\u26E7\u200D', |
| '84\U000E01D7\U0001D23B.\u200D\U000102F5\u26E7\u200D', |
| '\u00A1', 'xn--7a', '\u19DA', 'xn--pkf', '\u2615', 'xn--53h', |
| '\U0001E937.\U00010B90\U0001E881\U00010E60\u0624', |
| '\U0001E937.\U00010B90\U0001E881\U00010E60\u0648\u0654', |
| '\U0001E915.\U00010B90\U0001E881\U00010E60\u0648\u0654', |
| '\U0001E915.\U00010B90\U0001E881\U00010E60\u0624', |
| 'xn--ve6h.xn--jgb1694kz0b2176a', |
| '\u00DF\u3002\U000102F3\u2D0C\u0FB8', |
| 'ss\u3002\U000102F3\u2D0C\u0FB8', |
| 'ss.xn--lgd921mvv0m', |
| 'ss.\U000102F3\u2D0C\u0FB8', |
| 'xn--zca.xn--lgd921mvv0m', |
| '\u00DF.\U000102F3\u2D0C\u0FB8', |
| '\u00DF\uFF61\U000102F3\u2D0C\u0FB8', |
| 'ss\uFF61\U000102F3\u2D0C\u0FB8', |
| '\u16AD\uFF61\U0001D320\u00DF\U00016AF1', |
| '\u16AD\u3002\U0001D320\u00DF\U00016AF1', |
| '\u16AD\u3002\U0001D320SS\U00016AF1', |
| '\u16AD\u3002\U0001D320ss\U00016AF1', |
| '\u16AD\u3002\U0001D320Ss\U00016AF1', |
| 'xn--hwe.xn--ss-ci1ub261a', |
| '\u16AD.\U0001D320ss\U00016AF1', |
| '\u16AD.\U0001D320SS\U00016AF1', |
| '\u16AD.\U0001D320Ss\U00016AF1', |
| 'xn--hwe.xn--zca4946pblnc', |
| '\u16AD.\U0001D320\u00DF\U00016AF1', |
| '\u16AD\uFF61\U0001D320SS\U00016AF1', |
| '\u16AD\uFF61\U0001D320ss\U00016AF1', |
| '\u16AD\uFF61\U0001D320Ss\U00016AF1', |
| '\u2D1A\U000102F8\U000E0104\u30025\uD7F6\u103A', |
| 'xn--ilj2659d.xn--5-dug9054m', |
| '\u2D1A\U000102F8.5\uD7F6\u103A', |
| '\u2D1A\U000102F8\U000E0104\u3002\U0001D7DD\uD7F6\u103A', |
| 'xn--9-mfs8024b.', |
| '9\u9681\u2BEE.', |
| 'xn--ss-4epx629f.xn--ifh802b6a', |
| 'ss\uAAF6\u18A5.\u22B6\u2D21\u2D16', |
| 'xn--pt9c.xn--0kjya', |
| '\U00010A57.\u2D09\u2D15', |
| '\uA5F7\U00011180.\u075D\U00010A52', |
| 'xn--ju8a625r.xn--hpb0073k', |
| '\u03C2.\u0641\u0645\u064A\U0001F79B1.', |
| '\u03A3.\u0641\u0645\u064A\U0001F79B1.', |
| '\u03C3.\u0641\u0645\u064A\U0001F79B1.', |
| 'xn--4xa.xn--1-gocmu97674d.', |
| 'xn--3xa.xn--1-gocmu97674d.', |
| 'xn--1-5bt6845n.', |
| '1\U0001DA19\u2E16.', |
| 'xn--84-s850a.xn--59h6326e', |
| '84\U0001D23B.\U000102F5\u26E7', |
| 'xn--r97c.', |
| '\U000102F7.', |
| |
| # These appear to be errors in the test vectors. All relate to incorrectly applying |
| # bidi rules across label boundaries. Appears independently confirmed |
| # at http://www.alvestrand.no/pipermail/idna-update/2017-January/007946.html |
| '0\u00E0.\u05D0', '0a\u0300.\u05D0', '0A\u0300.\u05D0', '0\u00C0.\u05D0', 'xn--0-sfa.xn--4db', |
| '\u00E0\u02c7.\u05D0', 'a\u0300\u02c7.\u05D0', 'A\u0300\u02c7.\u05D0', '\u00C0\u02c7.\u05D0', |
| 'xn--0ca88g.xn--4db', '0A.\u05D0', '0a.\u05D0', '0a.xn--4db', 'c.xn--0-eha.xn--4db', |
| 'c.0\u00FC.\u05D0', 'c.0u\u0308.\u05D0', 'C.0U\u0308.\u05D0', 'C.0\u00DC.\u05D0', |
| 'C.0\u00FC.\u05D0', 'C.0\u0075\u0308.\u05D0', '\u06B6\u06DF\u3002\u2087\uA806', '\u06B6\u06DF\u30027\uA806', |
| 'xn--pkb6f.xn--7-x93e', '\u06B6\u06DF.7\uA806', '1.\uAC7E6.\U00010C41\u06D0', |
| '1.\u1100\u1165\u11B56.\U00010C41\u06D0', '1.xn--6-945e.xn--glb1794k', |
| ] |
| |
| def unicode_fixup(string): |
| """Replace backslash-u-XXXX with appropriate unicode characters.""" |
| return _RE_SURROGATE.sub(lambda match: chr( |
| (ord(match.group(0)[0]) - 0xd800) * 0x400 + |
| ord(match.group(0)[1]) - 0xdc00 + 0x10000), |
| _RE_UNICODE.sub(lambda match: chr(int(match.group(1), 16)), string)) |
| |
| def parse_idna_test_table(inputstream): |
| """Parse IdnaTestV2.txt and return a list of tuples.""" |
| for lineno, line in enumerate(inputstream): |
| line = line.decode('utf-8').strip() |
| if '#' in line: |
| line = line.split('#', 1)[0] |
| if not line: |
| continue |
| yield((lineno + 1, tuple(field.strip() for field in line.split(';')))) |
| |
| |
| class TestIdnaTest(unittest.TestCase): |
| """Run one of the IdnaTestV2.txt test lines.""" |
| def __init__(self, lineno=None, fields=None): |
| super().__init__() |
| self.lineno = lineno |
| self.fields = fields |
| |
| def id(self): |
| return '{}.{}'.format(super().id(), self.lineno) |
| |
| def shortDescription(self): |
| if not self.fields: |
| return '' |
| return 'IdnaTestV2.txt line {}: {}'.format(self.lineno, '; '.join(self.fields)) |
| |
| def runTest(self): |
| if not self.fields: |
| return '' |
| source, to_unicode, to_unicode_status, to_ascii, to_ascii_status, to_ascii_t, to_ascii_t_status = self.fields |
| if source in _SKIP_TESTS: |
| return |
| if not to_unicode: |
| to_unicode = source |
| if not to_unicode_status: |
| to_unicode_status = '[]' |
| if not to_ascii: |
| to_ascii = to_unicode |
| if not to_ascii_status: |
| to_ascii_status = to_unicode_status |
| if not to_ascii_t: |
| to_ascii_t = to_ascii |
| if not to_ascii_t_status: |
| to_ascii_t_status = to_ascii_status |
| |
| try: |
| output = idna.decode(source, uts46=True, strict=True) |
| if to_unicode_status != '[]': |
| self.fail('decode() did not emit required error {} for {}'.format(to_unicode, repr(source))) |
| self.assertEqual(output, to_unicode, 'unexpected decode() output') |
| except (idna.IDNAError, UnicodeError, ValueError) as exc: |
| if str(exc).startswith("Unknown"): |
| raise unittest.SkipTest("Test requires support for a newer" |
| " version of Unicode than this Python supports") |
| if to_unicode_status == '[]': |
| raise |
| |
| try: |
| output = idna.encode(source, uts46=True, strict=True).decode('ascii') |
| if to_ascii_status != '[]': |
| self.fail('encode() did not emit required error {} for {}'. |
| format(to_ascii_status, repr(source))) |
| self.assertEqual(output, to_ascii, 'unexpected encode() output') |
| except (idna.IDNAError, UnicodeError, ValueError) as exc: |
| if str(exc).startswith("Unknown"): |
| raise unittest.SkipTest("Test requires support for a newer" |
| " version of Unicode than this Python supports") |
| if to_ascii_status == '[]': |
| raise |
| |
| try: |
| output = idna.encode(source, uts46=True, strict=True, transitional=True).decode('ascii') |
| if to_ascii_t_status != '[]': |
| self.fail('encode(transitional=True) did not emit required error {} for {}'. |
| format(to_ascii_t_status, repr(source))) |
| self.assertEqual(output, to_ascii_t, 'unexpected encode() output') |
| except (idna.IDNAError, UnicodeError, ValueError) as exc: |
| if str(exc).startswith("Unknown"): |
| raise unittest.SkipTest("Test requires support for a newer" |
| " version of Unicode than this Python supports") |
| if to_ascii_t_status == '[]': |
| raise |
| |
| |
| def load_tests(loader, tests, pattern): |
| """Create a suite of all the individual tests.""" |
| suite = unittest.TestSuite() |
| with open(os.path.join(os.path.dirname(__file__), |
| 'IdnaTestV2.txt'), 'rb') as tests_file: |
| suite.addTests(TestIdnaTest(lineno, fields) |
| for lineno, fields in parse_idna_test_table(tests_file)) |
| return suite |
