Revert Unicode 16.0.0 data updates

Due to changes in the UTS46 processing logic, this is going to require a bit more work to implement. Since there is collateral impacts, reverting back to data from Unicode 15.1.0 for now.
kjd · Sep 15, 2024 · ceca619 · ceca619
1 parent c43ac75
commit ceca619
Show file tree

Hide file tree

Showing 3 changed files with 746 additions and 1,010 deletions.
diff --git a/idna/core.py b/idna/core.py
@@ -9,45 +9,6 @@
 _virama_combining_class = 9
 _alabel_prefix = b"xn--"
 _unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]")
-_ldh = (
-    48,
-    49,
-    50,
-    51,
-    52,
-    53,
-    54,
-    55,
-    56,
-    57,
-    95,
-    97,
-    98,
-    99,
-    100,
-    101,
-    102,
-    103,
-    104,
-    105,
-    106,
-    107,
-    108,
-    109,
-    110,
-    111,
-    112,
-    113,
-    114,
-    115,
-    116,
-    117,
-    118,
-    119,
-    120,
-    121,
-    122,
-)
 
 
 class IDNAError(UnicodeError):
@@ -380,16 +341,17 @@ def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False
             uts46row = uts46data[code_point if code_point < 256 else bisect.bisect_left(uts46data, (code_point, "Z")) - 1]
             status = uts46row[1]
             replacement: Optional[str] = None
-            if std3_rules and code_point <= 0x7F:
-                if code_point not in _ldh:
-                    raise InvalidCodepoint(
-                        "Codepoint {} at position {} does not follow STD3 rules".format(_unot(code_point), pos + 1)
-                    )
             if len(uts46row) == 3:
                 replacement = uts46row[2]
-            if status == "V" or (status == "D" and not transitional):
+            if (
+                status == "V"
+                or (status == "D" and not transitional)
+                or (status == "3" and not std3_rules and replacement is None)
+            ):
                 output += char
-            elif replacement is not None and (status == "M" or (status == "D" and transitional)):
+            elif replacement is not None and (
+                status == "M" or (status == "3" and not std3_rules) or (status == "D" and transitional)
+            ):
                 output += replacement
             elif status != "I":
                 raise IndexError()

diff --git a/idna/idnadata.py b/idna/idnadata.py
@@ -1,7 +1,6 @@
 # This file is automatically generated by tools/idna-data
 
-__version__ = "16.0.0"
-
+__version__ = "15.1.0"
 scripts = {
     "Greek": (
         0x37000000374,
@@ -728,7 +727,6 @@
     0x88C: 68,
     0x88D: 68,
     0x88E: 82,
-    0x897: 84,
     0x898: 84,
     0x899: 84,
     0x89A: 84,
@@ -1875,17 +1873,8 @@
     0x10D25: 84,
     0x10D26: 84,
     0x10D27: 84,
-    0x10D69: 84,
-    0x10D6A: 84,
-    0x10D6B: 84,
-    0x10D6C: 84,
-    0x10D6D: 84,
     0x10EAB: 84,
     0x10EAC: 84,
-    0x10EC2: 82,
-    0x10EC3: 68,
-    0x10EC4: 68,
-    0x10EFC: 84,
     0x10EFD: 84,
     0x10EFE: 84,
     0x10EFF: 84,
@@ -2064,17 +2053,6 @@
     0x11372: 84,
     0x11373: 84,
     0x11374: 84,
-    0x113BB: 84,
-    0x113BC: 84,
-    0x113BD: 84,
-    0x113BE: 84,
-    0x113BF: 84,
-    0x113C0: 84,
-    0x113CE: 84,
-    0x113D0: 84,
-    0x113D2: 84,
-    0x113E1: 84,
-    0x113E2: 84,
     0x11438: 84,
     0x11439: 84,
     0x1143A: 84,
@@ -2130,6 +2108,7 @@
     0x116B5: 84,
     0x116B7: 84,
     0x1171D: 84,
+    0x1171E: 84,
     0x1171F: 84,
     0x11722: 84,
     0x11723: 84,
@@ -2286,7 +2265,6 @@
     0x11F3A: 84,
     0x11F40: 84,
     0x11F42: 84,
-    0x11F5A: 84,
     0x13430: 84,
     0x13431: 84,
     0x13432: 84,
@@ -2319,21 +2297,6 @@
     0x13453: 84,
     0x13454: 84,
     0x13455: 84,
-    0x1611E: 84,
-    0x1611F: 84,
-    0x16120: 84,
-    0x16121: 84,
-    0x16122: 84,
-    0x16123: 84,
-    0x16124: 84,
-    0x16125: 84,
-    0x16126: 84,
-    0x16127: 84,
-    0x16128: 84,
-    0x16129: 84,
-    0x1612D: 84,
-    0x1612E: 84,
-    0x1612F: 84,
     0x16AF0: 84,
     0x16AF1: 84,
     0x16AF2: 84,
@@ -2642,8 +2605,6 @@
     0x1E4ED: 84,
     0x1E4EE: 84,
     0x1E4EF: 84,
-    0x1E5EE: 84,
-    0x1E5EF: 84,
     0x1E8D0: 84,
     0x1E8D1: 84,
     0x1E8D2: 84,
@@ -3367,7 +3328,7 @@
         0x8600000086B,
         0x87000000888,
         0x8890000088F,
-        0x897000008E2,
+        0x898000008E2,
         0x8E300000958,
         0x96000000964,
         0x96600000970,
@@ -3602,7 +3563,6 @@
         0x1C0000001C38,
         0x1C4000001C4A,
         0x1C4D00001C7E,
-        0x1C8A00001C8B,
         0x1CD000001CD3,
         0x1CD400001CFB,
         0x1D0000001D2C,
@@ -3966,13 +3926,11 @@
         0xA7C30000A7C4,
         0xA7C80000A7C9,
         0xA7CA0000A7CB,
-        0xA7CD0000A7CE,
         0xA7D10000A7D2,
         0xA7D30000A7D4,
         0xA7D50000A7D6,
         0xA7D70000A7D8,
         0xA7D90000A7DA,
-        0xA7DB0000A7DC,
         0xA7F60000A7F8,
         0xA7FA0000A828,
         0xA82C0000A82D,
@@ -4042,7 +4000,6 @@
         0x105A3000105B2,
         0x105B3000105BA,
         0x105BB000105BD,
-        0x105C0000105F4,
         0x1060000010737,
         0x1074000010756,
         0x1076000010768,
@@ -4080,14 +4037,10 @@
         0x10CC000010CF3,
         0x10D0000010D28,
         0x10D3000010D3A,
-        0x10D4000010D50,
-        0x10D6900010D6E,
-        0x10D6F00010D86,
         0x10E8000010EAA,
         0x10EAB00010EAD,
         0x10EB000010EB2,
-        0x10EC200010EC5,
-        0x10EFC00010F1D,
+        0x10EFD00010F1D,
         0x10F2700010F28,
         0x10F3000010F51,
         0x10F7000010F86,
@@ -4133,16 +4086,6 @@
         0x1135D00011364,
         0x113660001136D,
         0x1137000011375,
-        0x113800001138A,
-        0x1138B0001138C,
-        0x1138E0001138F,
-        0x11390000113B6,
-        0x113B7000113C1,
-        0x113C2000113C3,
-        0x113C5000113C6,
-        0x113C7000113CB,
-        0x113CC000113D4,
-        0x113E1000113E3,
         0x114000001144B,
         0x114500001145A,
         0x1145E00011462,
@@ -4157,7 +4100,6 @@
         0x116500001165A,
         0x11680000116B9,
         0x116C0000116CA,
-        0x116D0000116E4,
         0x117000001171B,
         0x1171D0001172C,
         0x117300001173A,
@@ -4181,8 +4123,6 @@
         0x11A5000011A9A,
         0x11A9D00011A9E,
         0x11AB000011AF9,
-        0x11BC000011BE1,
-        0x11BF000011BFA,
         0x11C0000011C09,
         0x11C0A00011C37,
         0x11C3800011C41,
@@ -4207,16 +4147,14 @@
         0x11F0000011F11,
         0x11F1200011F3B,
         0x11F3E00011F43,
-        0x11F5000011F5B,
+        0x11F5000011F5A,
         0x11FB000011FB1,
         0x120000001239A,
         0x1248000012544,
         0x12F9000012FF1,
         0x1300000013430,
         0x1344000013456,
-        0x13460000143FB,
         0x1440000014647,
-        0x161000001613A,
         0x1680000016A39,
         0x16A4000016A5F,
         0x16A6000016A6A,
@@ -4229,8 +4167,6 @@
         0x16B5000016B5A,
         0x16B6300016B78,
         0x16B7D00016B90,
-        0x16D4000016D6D,
-        0x16D7000016D7A,
         0x16E6000016E80,
         0x16F0000016F4B,
         0x16F4F00016F88,
@@ -4240,7 +4176,7 @@
         0x16FF000016FF2,
         0x17000000187F8,
         0x1880000018CD6,
-        0x18CFF00018D09,
+        0x18D0000018D09,
         0x1AFF00001AFF4,
         0x1AFF50001AFFC,
         0x1AFFD0001AFFF,
@@ -4255,7 +4191,6 @@
         0x1BC800001BC89,
         0x1BC900001BC9A,
         0x1BC9D0001BC9F,
-        0x1CCF00001CCFA,
         0x1CF000001CF2E,
         0x1CF300001CF47,
         0x1DA000001DA37,
@@ -4279,7 +4214,6 @@
         0x1E2900001E2AF,
         0x1E2C00001E2FA,
         0x1E4D00001E4FA,
-        0x1E5D00001E5FB,
         0x1E7E00001E7E7,
         0x1E7E80001E7EC,
         0x1E7ED0001E7EF,