Merge pull request #630 from sarubi/zwj_fix

Prevent Zero Width Joiner replaced with whitespace
google · Feb 25, 2021 · bc53923 · bc53923
2 parents d429804 + c970ded
commit bc53923
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 5 deletions.
diff --git a/data/nmt_nfkc.tsv b/data/nmt_nfkc.tsv
@@ -57263,8 +57263,7 @@ FB9	F90 FB5	# ྐྵ => ྐྵ
 200A	20	#   =>  
 200B	20	#  =>  
 200C	20	# ‌ =>  
-200D	20	# ‍ =>  
-200E	20	# ‎ =>  
+200E	20	# ‎ =>
 200F	20	# ‏ =>  
 2011	2010	# ‑ => ‐
 2017	20 333	# ‗ =>  ̳

diff --git a/data/nmt_nfkc_cf.tsv b/data/nmt_nfkc_cf.tsv
@@ -57980,8 +57980,7 @@ FB9	F90 FB5	# ྐྵ => ྐྵ
 200A	20	#   =>  
 200B	20	#  =>  
 200C	20	# ‌ =>  
-200D	20	# ‍ =>  
-200E	20	# ‎ =>  
+200E	20	# ‎ =>
 200F	20	# ‏ =>  
 2011	2010	# ‑ => ‐
 2017	20 333	# ‗ =>  ̳

diff --git a/src/builder.cc b/src/builder.cc
@@ -368,7 +368,6 @@ util::Status Builder::BuildNmtNFKCMap(CharsMap *chars_map) {
   nfkc_map[{0xFEFF}] = {0x20};  // ZERO WIDTH NO-BREAK
   nfkc_map[{0xFFFD}] = {0x20};  // REPLACEMENT CHARACTER
   nfkc_map[{0x200C}] = {0x20};  // ZERO WIDTH NON-JOINER
-  nfkc_map[{0x200D}] = {0x20};  // ZERO WIDTH JOINER
 
   // Ascii Control characters
   nfkc_map[{0x0001}] = {};