Merge pull request #190 from kjd/unicode-16

Unicode 16

Author: kjd

idna/core.py

@@ -9,6 +9,45 @@
 _virama_combining_class = 9
 _alabel_prefix = b"xn--"
 _unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]")
+_ldh = (
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57,
+    95,
+    97,
+    98,
+    99,
+    100,
+    101,
+    102,
+    103,
+    104,
+    105,
+    106,
+    107,
+    108,
+    109,
+    110,
+    111,
+    112,
+    113,
+    114,
+    115,
+    116,
+    117,
+    118,
+    119,
+    120,
+    121,
+    122,
+)
 
 
 class IDNAError(UnicodeError):
@@ -341,17 +380,16 @@ def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False
             uts46row = uts46data[code_point if code_point < 256 else bisect.bisect_left(uts46data, (code_point, "Z")) - 1]
             status = uts46row[1]
             replacement: Optional[str] = None
+            if std3_rules and code_point <= 0x7F:
+                if code_point not in _ldh:
+                    raise InvalidCodepoint(
+                        "Codepoint {} at position {} does not follow STD3 rules".format(_unot(code_point), pos + 1)
+                    )
             if len(uts46row) == 3:
                 replacement = uts46row[2]
-            if (
-                status == "V"
-                or (status == "D" and not transitional)
-                or (status == "3" and not std3_rules and replacement is None)
-            ):
+            if status == "V" or (status == "D" and not transitional):
                 output += char
-            elif replacement is not None and (
-                status == "M" or (status == "3" and not std3_rules) or (status == "D" and transitional)
-            ):
+            elif replacement is not None and (status == "M" or (status == "D" and transitional)):
                 output += replacement
             elif status != "I":
                 raise IndexError()

idna/idnadata.py

@@ -1,6 +1,7 @@
 # This file is automatically generated by tools/idna-data
 
-__version__ = "15.1.0"
+__version__ = "16.0.0"
+
 scripts = {
     "Greek": (
         0x37000000374,
@@ -727,6 +728,7 @@
     0x88C: 68,
     0x88D: 68,
     0x88E: 82,
+    0x897: 84,
     0x898: 84,
     0x899: 84,
     0x89A: 84,
@@ -1873,8 +1875,17 @@
     0x10D25: 84,
     0x10D26: 84,
     0x10D27: 84,
+    0x10D69: 84,
+    0x10D6A: 84,
+    0x10D6B: 84,
+    0x10D6C: 84,
+    0x10D6D: 84,
     0x10EAB: 84,
     0x10EAC: 84,
+    0x10EC2: 82,
+    0x10EC3: 68,
+    0x10EC4: 68,
+    0x10EFC: 84,
     0x10EFD: 84,
     0x10EFE: 84,
     0x10EFF: 84,
@@ -2053,6 +2064,17 @@
     0x11372: 84,
     0x11373: 84,
     0x11374: 84,
+    0x113BB: 84,
+    0x113BC: 84,
+    0x113BD: 84,
+    0x113BE: 84,
+    0x113BF: 84,
+    0x113C0: 84,
+    0x113CE: 84,
+    0x113D0: 84,
+    0x113D2: 84,
+    0x113E1: 84,
+    0x113E2: 84,
     0x11438: 84,
     0x11439: 84,
     0x1143A: 84,
@@ -2108,7 +2130,6 @@
     0x116B5: 84,
     0x116B7: 84,
     0x1171D: 84,
-    0x1171E: 84,
     0x1171F: 84,
     0x11722: 84,
     0x11723: 84,
@@ -2265,6 +2286,7 @@
     0x11F3A: 84,
     0x11F40: 84,
     0x11F42: 84,
+    0x11F5A: 84,
     0x13430: 84,
     0x13431: 84,
     0x13432: 84,
@@ -2297,6 +2319,21 @@
     0x13453: 84,
     0x13454: 84,
     0x13455: 84,
+    0x1611E: 84,
+    0x1611F: 84,
+    0x16120: 84,
+    0x16121: 84,
+    0x16122: 84,
+    0x16123: 84,
+    0x16124: 84,
+    0x16125: 84,
+    0x16126: 84,
+    0x16127: 84,
+    0x16128: 84,
+    0x16129: 84,
+    0x1612D: 84,
+    0x1612E: 84,
+    0x1612F: 84,
     0x16AF0: 84,
     0x16AF1: 84,
     0x16AF2: 84,
@@ -2605,6 +2642,8 @@
     0x1E4ED: 84,
     0x1E4EE: 84,
     0x1E4EF: 84,
+    0x1E5EE: 84,
+    0x1E5EF: 84,
     0x1E8D0: 84,
     0x1E8D1: 84,
     0x1E8D2: 84,
@@ -3328,7 +3367,7 @@
         0x8600000086B,
         0x87000000888,
         0x8890000088F,
-        0x898000008E2,
+        0x897000008E2,
         0x8E300000958,
         0x96000000964,
         0x96600000970,
@@ -3563,6 +3602,7 @@
         0x1C0000001C38,
         0x1C4000001C4A,
         0x1C4D00001C7E,
+        0x1C8A00001C8B,
         0x1CD000001CD3,
         0x1CD400001CFB,
         0x1D0000001D2C,
@@ -3926,11 +3966,13 @@
         0xA7C30000A7C4,
         0xA7C80000A7C9,
         0xA7CA0000A7CB,
+        0xA7CD0000A7CE,
         0xA7D10000A7D2,
         0xA7D30000A7D4,
         0xA7D50000A7D6,
         0xA7D70000A7D8,
         0xA7D90000A7DA,
+        0xA7DB0000A7DC,
         0xA7F60000A7F8,
         0xA7FA0000A828,
         0xA82C0000A82D,
@@ -4000,6 +4042,7 @@
         0x105A3000105B2,
         0x105B3000105BA,
         0x105BB000105BD,
+        0x105C0000105F4,
         0x1060000010737,
         0x1074000010756,
         0x1076000010768,
@@ -4037,10 +4080,14 @@
         0x10CC000010CF3,
         0x10D0000010D28,
         0x10D3000010D3A,
+        0x10D4000010D50,
+        0x10D6900010D6E,
+        0x10D6F00010D86,
         0x10E8000010EAA,
         0x10EAB00010EAD,
         0x10EB000010EB2,
-        0x10EFD00010F1D,
+        0x10EC200010EC5,
+        0x10EFC00010F1D,
         0x10F2700010F28,
         0x10F3000010F51,
         0x10F7000010F86,
@@ -4086,6 +4133,16 @@
         0x1135D00011364,
         0x113660001136D,
         0x1137000011375,
+        0x113800001138A,
+        0x1138B0001138C,
+        0x1138E0001138F,
+        0x11390000113B6,
+        0x113B7000113C1,
+        0x113C2000113C3,
+        0x113C5000113C6,
+        0x113C7000113CB,
+        0x113CC000113D4,
+        0x113E1000113E3,
         0x114000001144B,
         0x114500001145A,
         0x1145E00011462,
@@ -4100,6 +4157,7 @@
         0x116500001165A,
         0x11680000116B9,
         0x116C0000116CA,
+        0x116D0000116E4,
         0x117000001171B,
         0x1171D0001172C,
         0x117300001173A,
@@ -4123,6 +4181,8 @@
         0x11A5000011A9A,
         0x11A9D00011A9E,
         0x11AB000011AF9,
+        0x11BC000011BE1,
+        0x11BF000011BFA,
         0x11C0000011C09,
         0x11C0A00011C37,
         0x11C3800011C41,
@@ -4147,14 +4207,16 @@
         0x11F0000011F11,
         0x11F1200011F3B,
         0x11F3E00011F43,
-        0x11F5000011F5A,
+        0x11F5000011F5B,
         0x11FB000011FB1,
         0x120000001239A,
         0x1248000012544,
         0x12F9000012FF1,
         0x1300000013430,
         0x1344000013456,
+        0x13460000143FB,
         0x1440000014647,
+        0x161000001613A,
         0x1680000016A39,
         0x16A4000016A5F,
         0x16A6000016A6A,
@@ -4167,6 +4229,8 @@
         0x16B5000016B5A,
         0x16B6300016B78,
         0x16B7D00016B90,
+        0x16D4000016D6D,
+        0x16D7000016D7A,
         0x16E6000016E80,
         0x16F0000016F4B,
         0x16F4F00016F88,
@@ -4176,7 +4240,7 @@
         0x16FF000016FF2,
         0x17000000187F8,
         0x1880000018CD6,
-        0x18D0000018D09,
+        0x18CFF00018D09,
         0x1AFF00001AFF4,
         0x1AFF50001AFFC,
         0x1AFFD0001AFFF,
@@ -4191,6 +4255,7 @@
         0x1BC800001BC89,
         0x1BC900001BC9A,
         0x1BC9D0001BC9F,
+        0x1CCF00001CCFA,
         0x1CF000001CF2E,
         0x1CF300001CF47,
         0x1DA000001DA37,
@@ -4214,6 +4279,7 @@
         0x1E2900001E2AF,
         0x1E2C00001E2FA,
         0x1E4D00001E4FA,
+        0x1E5D00001E5FB,
         0x1E7E00001E7E7,
         0x1E7E80001E7EC,
         0x1E7ED0001E7EF,