Disable japanese-transliteration by default

meilisearch#149 (review) meilisearch#131 (comment) Co-authored-by: ManyTheFish <[email protected]>
choznerol · Oct 17, 2022 · dfcaa62 · dfcaa62
1 parent 49bc172
commit dfcaa62
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 6 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -21,8 +21,10 @@ slice-group-by = "0.3.0"
 unicode-segmentation = "1.6.0"
 whatlang = "0.16.1"
 lindera = { version = "=0.16.0", features = ["ipadic"], optional = true }
-pinyin = { version = "0.9", default-features = false, features = ["with_tone"], optional = true }
-wana_kana = "2.1.0"
+pinyin = { version = "0.9", default-features = false, features = [
+  "with_tone",
+], optional = true }
+wana_kana = { version = "2.1.0", optional = true }
 
 [features]
 default = ["chinese", "hebrew", "japanese", "thai"]
@@ -35,6 +37,7 @@ hebrew = []
 
 # allow japanese specialized tokenization
 japanese = ["dep:lindera"]
+japanese-transliteration = ["dep:wana_kana"]
 
 # allow thai specialized tokenization
 thai = []

diff --git a/src/normalizer/mod.rs b/src/normalizer/mod.rs
@@ -3,7 +3,7 @@ use once_cell::sync::Lazy;
 #[cfg(feature = "chinese")]
 pub use self::chinese::ChineseNormalizer;
 pub use self::control_char::ControlCharNormalizer;
-#[cfg(feature = "japanese")]
+#[cfg(feature = "japanese-transliteration")]
 pub use self::japanese::JapaneseNormalizer;
 pub use self::latin::LatinNormalizer;
 pub use self::lowercase::LowercaseNormalizer;
@@ -14,7 +14,7 @@ use crate::Token;
 #[cfg(feature = "chinese")]
 mod chinese;
 mod control_char;
-#[cfg(feature = "japanese")]
+#[cfg(feature = "japanese-transliteration")]
 mod japanese;
 mod latin;
 mod lowercase;
@@ -26,7 +26,7 @@ pub static NORMALIZERS: Lazy<Vec<Box<dyn Normalizer>>> = Lazy::new(|| {
         Box::new(LowercaseNormalizer),
         #[cfg(feature = "chinese")]
         Box::new(ChineseNormalizer),
-        #[cfg(feature = "japanese")]
+        #[cfg(feature = "japanese-transliteration")]
         Box::new(JapaneseNormalizer),
         Box::new(LatinNormalizer),
         Box::new(ControlCharNormalizer),

diff --git a/src/segmenter/japanese.rs b/src/segmenter/japanese.rs
@@ -49,7 +49,8 @@ mod test {
         "国際",
         "空港",
         "限定",
-        "とうとばっぐ",
+        // Use "とうとばっぐ" instead when feature "japanese-transliteration" is enabled or become default
+        "トートバッグ",
         " ",
         "すもも",
         "も",