[Misc] Upgrade Tantivy to 0.21.0.

Cargo.toml

@@ -44,7 +44,7 @@ serde_json = "1.0.105"
 serde_repr = "0.1.16"
 serde_with = "3.3.0"
 sqlx = "0.7.1"
-tantivy = "0.20.2"
+tantivy = "0.21.0"
 time = "0.3.28"
 tlsh2 = "0.3.0"
 tokio-cron-scheduler = "0.9.4"

src/main.rs

@@ -61,7 +61,7 @@ fn process_command(version: &str, matches: ArgMatches) -> Result<(), anyhow::Err
                     Url::parse(url)
                         .with_context(|| "Cannot parse Web Scraper URL parameter.".to_string())
                 })?,
-            search_index_version: 2,
+            search_index_version: 3,
         },
         jobs: SchedulerJobsConfig {
             resources_trackers_schedule: matches
@@ -412,7 +412,7 @@ mod tests {
             }),
             components: ComponentsConfig {
                 web_scraper_url: Url::parse("http://localhost:7272")?,
-                search_index_version: 2,
+                search_index_version: 3,
             },
             jobs: SchedulerJobsConfig {
                 resources_trackers_schedule: Schedule::try_from("0 * 0 * * * *")?,

src/search/search_index.rs

@@ -90,16 +90,16 @@ impl SearchIndex {
         let (schema_fields, schema) = SearchIndexSchemaFields::build();
         let index = initializer(schema)?;
 
+        let ids_tokenizer = TextAnalyzer::builder(RawTokenizer::default())
+            .filter(LowerCaser)
+            .build();
+
         let tokenizers = index.tokenizers();
-        tokenizers.register("ngram2_10", NgramTokenizer::prefix_only(2, 10));
+        tokenizers.register("ngram2_10", NgramTokenizer::prefix_only(2, 10)?);
+        tokenizers.register("ids", ids_tokenizer.clone());
 
         let tokenizers = index.fast_field_tokenizer();
-        tokenizers.register(
-            "ids",
-            TextAnalyzer::builder(RawTokenizer::default())
-                .filter(LowerCaser)
-                .build(),
-        );
+        tokenizers.register("ids", ids_tokenizer);
 
         let index_reader = index
             .reader_builder()