dotnet · tqiu8 · Apr 2, 2021 · Apr 12, 2021 · Apr 14, 2021 · Apr 14, 2021
diff --git a/SHARDS.md b/SHARDS.md
@@ -0,0 +1,48 @@
+# Building ICU Shards
+
+The ICU is divided by locales and features. <br/>
+
+By locale:
+- EFIGS (en, fr, it, de, es)
+- CJK (zh, ja, ko)
+- no CJK (all locales except for zh, ja, ko)
+
+By features:
+- Collation
+- Normalization
+- Currency
+- Locales
+- Zones
+
+To generate ICU shards run:
+
+`make -C ./eng -f icu.mk shards`
+
+which will build all shards from filter files available in `icu-filters/` as well as `icu-dictionary.json`, which maps each parent locale (i.e. en) to relevant files.
+
+## ICU dictionary
+The ICU dictionary is divided into the following format:
+
+```
+{
+  "en": {
+    "essentials": [ 
+      icudt_currency.dat,
+      icudt_normalization.dat,
+      icudt_base.dat
+    ]
+    "zones": [ relevant timezone data files ],
+    "locales": [ relevant locale data files ],
+    "coll": [ relevant collationd data files ]
+  }
+  .
+  .
+  .
+}
+```
+
+To generate just the ICU dictionary run:
+
+`make -C ./eng -f icu.mk icu_dictionary.json`
+
+The dictionary is packaged with the data files to be consumed later by the WASM runtime.
diff --git a/eng/icu.mk b/eng/icu.mk
@@ -94,6 +94,30 @@ $(eval $(call TargetBuildTemplate,icudt))
 $(eval $(call TargetBuildTemplate,icudt_CJK))
 $(eval $(call TargetBuildTemplate,icudt_no_CJK))
 $(eval $(call TargetBuildTemplate,icudt_EFIGS))
+$(eval $(call TargetBuildTemplate,icudt_normalization))
+$(eval $(call TargetBuildTemplate,icudt_base))
+$(eval $(call TargetBuildTemplate,icudt_efigs_locales))
+$(eval $(call TargetBuildTemplate,icudt_currency))
+$(eval $(call TargetBuildTemplate,icudt_coll))
+$(eval $(call TargetBuildTemplate,icudt_zones))
+$(eval $(call TargetBuildTemplate,icudt_cjk_zones))
+$(eval $(call TargetBuildTemplate,icudt_no_cjk_zones))
+$(eval $(call TargetBuildTemplate,icudt_efigs_zones))
+$(eval $(call TargetBuildTemplate,icudt_cjk_locales))
+$(eval $(call TargetBuildTemplate,icudt_no_cjk_locales))
+$(eval $(call TargetBuildTemplate,icudt_locales))
+$(eval $(call TargetBuildTemplate,icudt_efigs_coll))
+$(eval $(call TargetBuildTemplate,icudt_cjk_coll))
+$(eval $(call TargetBuildTemplate,icudt_no_cjk_coll))
+
+ICU_SHARDS := icudt_base icudt_normalization icudt_currency icudt_coll icudt_zones icudt_cjk_zones icudt_no_cjk_zones icudt_efigs_zones icudt_efigs_locales icudt_cjk_locales icudt_no_cjk_locales icudt_locales icudt_efigs_coll icudt_cjk_coll icudt_no_cjk_coll
+DATA_SHARDS := $(addprefix data-, $(ICU_SHARDS))
 
 # build source+data for the main "icudt" filter and only data for the other filters
 all: lib-icudt data-icudt data-icudt_no_CJK data-icudt_EFIGS data-icudt_CJK
+
+icu_dictionary.json:
+	cd $(TOP)/icu/icu4c/source/ && PYTHONPATH=python python3 -m icutools.databuilder --mode=makedict --filter_file=$(ICU_FILTER_PATH)/icudt.json \
+	&& mv icu_dictionary.json $(TARGET_BINDIR)
+
+shards: $(DATA_SHARDS) icu_dictionary.json
diff --git a/icu-filters/curr-root.txt b/icu-filters/curr-root.txt
@@ -196,4 +196,4 @@ root{
             surroundingMatch{"[:digit:]"}
         }
     }
-}
+}
diff --git a/icu-filters/icudt_base.json b/icu-filters/icudt_base.json
@@ -0,0 +1,242 @@
+{
+    "localeFilter": {
+        "filterType": "locale",
+        "includeScripts": false,	
+        "includeChildren": false,
+        "whitelist": [
+            "ar_SA",
+            "am_ET",
+            "bg_BG",
+            "bn_BD",
+            "bn_IN",
+            "ca_AD",
+            "ca_ES",
+            "cs_CZ",
+            "da_DK",
+            "de_AT",
+            "de_BE",
+            "de_CH",
+            "de_DE",
+            "de_IT",
+            "de_LI",
+            "de_LU",
+            "el_CY",
+            "el_GR",
+            "en_AE",
+            "en_AG",
+            "en_AI",
+            "en_AS",
+            "en_AT",
+            "en_AU",
+            "en_BB",
+            "en_BE",
+            "en_BI",
+            "en_BM",
+            "en_BS",
+            "en_BW",
+            "en_BZ",
+            "en_CA",
+            "en_CC",
+            "en_CH",
+            "en_CK",
+            "en_CM",
+            "en_CX",
+            "en_CY",
+            "en_DE",
+            "en_DK",
+            "en_DM",
+            "en_ER",
+            "en_FI",
+            "en_FJ",
+            "en_FK",
+            "en_FM",
+            "en_GB",
+            "en_GD",
+            "en_GG",
+            "en_GH",
+            "en_GI",
+            "en_GM",
+            "en_GU",
+            "en_GY",
+            "en_HK",
+            "en_IE",
+            "en_IL",
+            "en_IM",
+            "en_IN",
+            "en_IO",
+            "en_JE",
+            "en_JM",
+            "en_KE",
+            "en_KI",
+            "en_KN",
+            "en_KY",
+            "en_LC",
+            "en_LR",
+            "en_LS",
+            "en_MG",
+            "en_MH",
+            "en_MO",
+            "en_MP",
+            "en_MS",
+            "en_MT",
+            "en_MU",
+            "en_MW",
+            "en_MY",
+            "en_NA",
+            "en_NF",
+            "en_NG",
+            "en_NL",
+            "en_NR",
+            "en_NU",
+            "en_NZ",
+            "en_PG",
+            "en_PH",
+            "en_PK",
+            "en_PN",
+            "en_PR",
+            "en_PW",
+            "en_RW",
+            "en_SB",
+            "en_SC",
+            "en_SD",
+            "en_SE",
+            "en_SG",
+            "en_SH",
+            "en_SI",
+            "en_SL",
+            "en_SS",
+            "en_SX",
+            "en_SZ",
+            "en_TC",
+            "en_TK",
+            "en_TO",
+            "en_TT",
+            "en_TV",
+            "en_TZ",
+            "en_UG",
+            "en_UM",
+            "en_US",
+            "en_VC",
+            "en_VG",
+            "en_VI",
+            "en_VU",
+            "en_WS",
+            "en_ZA",
+            "en_ZM",
+            "en_ZW",
+            "en_US",
+            "es_419",
+            "es_ES",
+            "es_MX",
+            "et_EE",
+            "fa_IR",
+            "fi_FI",
+            "fil_PH",
+            "fr_BE",
+            "fr_CA",
+            "fr_CH",
+            "fr_FR",
+            "gu_IN",
+            "he_IL",
+            "hi_IN",
+            "hr_BA",
+            "hr_HR",
+            "hu_HU",
+            "id_ID",
+            "it_CH",
+            "it_IT",
+            "ja_JP",
+            "kn_IN",
+            "ko_KR",
+            "lt_LT",
+            "lv_LV",
+            "ml_IN",
+            "mr_IN",
+            "ms_BN",
+            "ms_MY",
+            "ms_SG",
+            "nl_AW",
+            "nl_BE",
+            "nl_NL",
+            "pl_PL",
+            "pt_BR",
+            "pt_PT",
+            "ro_RO",
+            "ru_RU",
+            "sk_SK",
+            "sl_SI",
+            "sr_Cyrl_RS",
+            "sr_Latn_RS",
+            "sv_AX",
+            "sv_SE",
+            "sw_CD",
+            "sw_KE",
+            "sw_TZ",
+            "sw_UG",
+            "ta_IN",
+            "ta_LK",
+            "ta_MY",
+            "ta_SG",
+            "te_IN",
+            "th_TH",
+            "tr_CY",
+            "tr_TR",
+            "uk_UA",
+            "vi_VN",
+            "zh_CN",
+            "zh_Hans_HK",
+            "zh_SG",
+            "zh_HK",
+            "zh_TW"
+        ]
+    },
+    "featureFilters": {
+        "locales_tree": "exclude",
+        "conversion_mappings": "exclude",
+        "confusables": "exclude",
+        "stringprep": "exclude",
+        "zone_supplemental": "exclude",
+        "zone_tree": "exclude",
+        "translit": "exclude",
+        "unames": "exclude",
+        "ulayout": "exclude",
+        "unit_tree": "exclude",
+        "rbnf_tree": "exclude",
+        "cnvalias": "exclude",
+        "lang_tree": "exclude",
+        "region_tree": "exclude",
+        "normalization": "exclude",
+        "coll_ucadata": "exclude",
+        "coll_tree": "exclude",
+        "misc": {
+            "whitelist": [
+                "numberingSystems",
+                "icuver",
+                "keyTypeData",
+                "supplementalData"
+            ]
+        },
+        "curr_tree": "exclude",
+        "curr_supplemental": "exclude",
+        "brkitr_dictionaries": "exclude",
+        "brkitr_rules": {
+            "whitelist": ["char"]
+        }
+    },
+    "resourceFilters": [
+        {
+            "categories": ["misc"],
+            "files": {
+                "whitelist": ["supplementalData"]
+            },
+            "rules": [
+                "-/*",
+                "+/calendarData",
+                "+/calendarPreferenceData",
+                "+/timeData",
+                "+/weekData",
+                "+/cldrVersion"     
+            ]
+        }
+    ]
+}
diff --git a/icu-filters/icudt_cjk_coll.json b/icu-filters/icudt_cjk_coll.json
@@ -0,0 +1,47 @@
+{
+  "collationUCAData": "implicithan",
+  "localeFilter": {
+      "filterType": "locale",
+      "includeScripts": false,	
+      "includeChildren": false,
+      "whitelist": [
+        "ja",
+        "ko",
+        "zh"
+      ]
+  },
+  "featureFilters": {
+      "locales_tree": "exclude",
+      "conversion_mappings": "exclude",
+      "confusables": "exclude",
+      "stringprep": "exclude",
+      "zone_tree": "exclude",
+      "zone_supplemental": "exclude",
+      "translit": "exclude",
+      "unames": "exclude",
+      "ulayout": "exclude",
+      "unit_tree": "exclude",
+      "rbnf_tree": "exclude",
+      "cnvalias": "exclude",
+      "lang_tree": "exclude",
+      "region_tree": "exclude",
+      "normalization": "exclude",
+      "misc": "exclude",
+      "curr_tree": "exclude",
+      "curr_supplemental": "exclude",
+      "brkitr_dictionaries": "exclude",
+      "brkitr_rules": "exclude"
+  },
+  "resourceFilters": [
+      {
+          "categories": ["coll_tree"],
+          "rules": [
+              "-/*/*",
+              "+/collations/default",
+              "+/collations/standard",
+              "+/collations/private-kana",
+              "-/UCARules"
+          ]
+      }
+  ]
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -196,4 +196,4 @@ root{ @@
                 surroundingMatch{"[:digit:]"}
             }
         }
-    }
+    }