feat: support search with lang cyrillic

.changeset/loud-rules-hang.md

@@ -0,0 +1,5 @@
+---
+"@rspress/theme-default": major
+---
+
+support search with lang cyrillic

packages/theme-default/src/components/Search/logic/providers/LocalProvider.ts

@@ -19,12 +19,15 @@ interface PageIndexForFlexSearch extends PageIndexInfo {
 
 const cjkRegex =
   /[\u3131-\u314e|\u314f-\u3163|\uac00-\ud7a3]|[\u4E00-\u9FCC\u3400-\u4DB5\uFA0E\uFA0F\uFA11\uFA13\uFA14\uFA1F\uFA21\uFA23\uFA24\uFA27-\uFA29]|[\ud840-\ud868][\udc00-\udfff]|\ud869[\udc00-\uded6\udf00-\udfff]|[\ud86a-\ud86c][\udc00-\udfff]|\ud86d[\udc00-\udf34\udf40-\udfff]|\ud86e[\udc00-\udc1d]|[\u3041-\u3096]|[\u30A1-\u30FA]/giu;
+const cyrillicRegex = /[\u0400-\u04FF]/g;
 
 export class LocalProvider implements Provider {
   #index?: SearchIndex<PageIndexInfo[]>;
 
   #cjkIndex?: SearchIndex<PageIndexInfo[]>;
 
+  #cyrilicIndex?: SearchIndex<PageIndexInfo[]>;
+
   async #getPages(lang: string): Promise<PageIndexInfo[]> {
     const result = await fetch(
       `${process.env.__ASSET_PREFIX__}/static/${SEARCH_INDEX_NAME}${
@@ -76,8 +79,23 @@ export class LocalProvider implements Provider {
         return cjkWords;
       },
     });
+    this.#cyrilicIndex = FlexSearch.create({
+      ...createOptions,
+      tokenize(str: string) {
+        const ciWords: string[] = [];
+        let m: RegExpExecArray | null = null;
+        do {
+          m = cyrillicRegex.exec(str);
+          if (m) {
+            ciWords.push(m[0]);
+          }
+        } while (m);
+        return ciWords;
+      },
+    });
     this.#index.add(pagesForSearch);
     this.#cjkIndex.add(pagesForSearch);
+    this.#cyrilicIndex.add(pagesForSearch);
   }
 
   async search(query: SearchQuery) {
@@ -91,6 +109,7 @@ export class LocalProvider implements Provider {
     const searchResult = await Promise.all([
       this.#index?.search(searchParams),
       this.#cjkIndex?.search(searchParams),
+      this.#cyrilicIndex.search(searchParams),
     ]);
     const flattenSearchResult = searchResult.flat(2).filter(Boolean);
 

packages/theme-default/src/components/Search/logic/util.ts

@@ -2,6 +2,7 @@ import { RemoteSearchIndexInfo, Header } from '@rspress/shared';
 
 const MAX_TITLE_LENGTH = 20;
 const kRegex = /[\u3131-\u314e|\u314f-\u3163|\uac00-\ud7a3]/u;
+const cyrillicRegex = /[\u0400-\u04FF]/u;
 
 export function backTrackHeaders(
   rawHeaders: Header[],
@@ -38,12 +39,16 @@ export function formatText(text: string) {
 }
 
 export function normalizeTextCase(text: string | number) {
-  const result = text
-    .toString()
-    .toLowerCase()
-    .normalize('NFD')
-    .replace(/[\u0300-\u036f]/g, '');
-  return kRegex.test(String(text)) ? result.normalize('NFC') : result;
+  const textNormalized = text.toString().toLowerCase().normalize('NFD');
+  const resultWithAccents = textNormalized;
+  const resultWithoutAccents = textNormalized.replace(/[\u0300-\u036f]/g, '');
+  if (cyrillicRegex.test(String(text))) {
+    return resultWithAccents.normalize('NFC');
+  }
+  if (kRegex.test(String(text))) {
+    return resultWithoutAccents.normalize('NFC');
+  }
+  return resultWithoutAccents;
 }
 
 export function removeDomain(url: string) {