ICU sharding.

SHARDS.md

@@ -0,0 +1,21 @@
+# Building ICU Shards
+
+The ICU is divided by locales and features. <br/>
+
+By locale:
+- EFIGS (en, fr, it, de, es)
+- CJK (zh, ja, ko)
+- no CJK (all locales except for zh, ja, ko)
+
+By features:
+- Collation
+- Normalization
+- Currency
+- Locales
+- Zones
+
+To generate ICU shards run:
+
+`make -C ./eng -f icu.mk shards`
+
+which will create `json` files in `icu-filters` and build the project with all shards from `icu-filters/`. Output files will be located in `artifacts/bin/icu-{platform}-{architecture}`.

eng/azure-pipelines.yml

@@ -32,7 +32,7 @@ stages:
       ############ BROWSER BUILD ############
       - job: Build_Browser
         displayName: Browser
-        timeoutInMinutes: 30
+        timeoutInMinutes: 50
         pool:
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             vmImage: ubuntu-20.04
@@ -43,7 +43,7 @@ stages:
           image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220531132048-00a561c
         steps:
         - bash: |
-            ./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm $(_InternalBuildArgs)
+            ./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:IcuSharding=true $(_InternalBuildArgs)
           displayName: Build
         - task: PublishPipelineArtifact@1
           displayName: Upload artifacts

eng/icu.browser.mk

@@ -1,5 +1,6 @@
 ENV_INIT_SCRIPT = source $(EMSDK_PATH)/emsdk_env.sh &&
 ENV_CONFIGURE_WRAPPER = emconfigure
+ICU_SHARDING = true
 
 ifeq ($(WASM_ENABLE_THREADS),true)
 	THREADS_FLAG="-pthread"

eng/icu.mk

@@ -99,6 +99,39 @@ endif
 $(eval $(call TargetBuildTemplate,icudt_CJK,icudt_CJK))
 $(eval $(call TargetBuildTemplate,icudt_no_CJK,icudt_no_CJK))
 $(eval $(call TargetBuildTemplate,icudt_EFIGS,icudt_EFIGS))
+$(eval $(call TargetBuildTemplate,icudt_normalization,icudt_normalization))
+$(eval $(call TargetBuildTemplate,icudt_base,icudt_base))
+$(eval $(call TargetBuildTemplate,icudt_currency,icudt_currency))
+$(eval $(call TargetBuildTemplate,icudt_full_full,icudt_full_full))
+$(eval $(call TargetBuildTemplate,icudt_full_coll,icudt_full_coll))
+$(eval $(call TargetBuildTemplate,icudt_full_zones,icudt_full_zones))
+$(eval $(call TargetBuildTemplate,icudt_full_locales,icudt_full_locales))
+$(eval $(call TargetBuildTemplate,icudt_efigs_locales,icudt_efigs_locales))
+$(eval $(call TargetBuildTemplate,icudt_efigs_coll,icudt_efigs_coll))
+$(eval $(call TargetBuildTemplate,icudt_efigs_full,icudt_efigs_full))
+$(eval $(call TargetBuildTemplate,icudt_efigs_zones,icudt_efigs_zones))
+$(eval $(call TargetBuildTemplate,icudt_cjk_locales,icudt_cjk_locales))
+$(eval $(call TargetBuildTemplate,icudt_cjk_zones,icudt_cjk_zones))
+$(eval $(call TargetBuildTemplate,icudt_cjk_full,icudt_cjk_full))
+$(eval $(call TargetBuildTemplate,icudt_cjk_coll,icudt_cjk_coll))
+$(eval $(call TargetBuildTemplate,icudt_no_cjk_full,icudt_no_cjk_full))
+$(eval $(call TargetBuildTemplate,icudt_no_cjk_coll,icudt_no_cjk_coll))
+$(eval $(call TargetBuildTemplate,icudt_no_cjk_zones,icudt_no_cjk_zones))
+$(eval $(call TargetBuildTemplate,icudt_no_cjk_locales,icudt_no_cjk_locales))
+
+
+ICU_SHARDS := icudt_base icudt_normalization icudt_currency icudt_cjk_zones icudt_no_cjk_zones icudt_efigs_zones icudt_full_zones icudt_efigs_locales icudt_cjk_locales icudt_no_cjk_locales icudt_full_locales icudt_efigs_coll icudt_cjk_coll icudt_no_cjk_coll icudt_full_coll icudt_efigs_full icudt_cjk_full icudt_no_cjk_full icudt_full_full
+DATA_SHARDS := $(addprefix data-, $(ICU_SHARDS))
+
+dictionary:
+	cd $(TOP)/icu/icu4c/source/ && PYTHONPATH=python python3 -m icutools.databuilder --mode=makedict --filter_dir=$(ICU_FILTER_PATH) --out_dir=$(TARGET_BINDIR) --shard_cfg=$(abspath $(CURDIR)/main-config.json) --exclude_feats=zones;
+
+shards: dictionary $(DATA_SHARDS) 
+
+BUILD_DATA := lib-icudt data-icudt data-icudt_no_CJK data-icudt_EFIGS data-icudt_CJK
+# sharded files are produced only when sharding is enabled
+ifeq ($(ICU_SHARDING), true)
+BUILD_DATA := $(BUILD_DATA) shards
+endif
 
-# build source+data for the main "icudt" filter and only data for the other filters
-all: lib-icudt data-icudt data-icudt_no_CJK data-icudt_EFIGS data-icudt_CJK
+all: $(BUILD_DATA)

eng/icu.proj

@@ -7,10 +7,10 @@
       <_ExtraParams Condition="'$(WasmEnableThreads)' == 'true'">WASM_ENABLE_THREADS=true</_ExtraParams>
     </PropertyGroup>
 
-    <Copy SourceFiles="$(MSBuildThisFileDirectory)\..\icu-filters\curr-root.txt" 
-          DestinationFiles="$(MSBuildThisFileDirectory)\..\icu\icu4c\source\data\curr\root.txt"/> 
+    <Copy SourceFiles="$(MSBuildThisFileDirectory)\..\icu-filters\curr-root.txt"
+          DestinationFiles="$(MSBuildThisFileDirectory)\..\icu\icu4c\source\data\curr\root.txt"/>
     <Exec WorkingDirectory="$(MSBuildThisFileDirectory)"
-          Command="make -f icu.mk all TARGET_OS=$(TargetOS.ToLower()) TARGET_ARCHITECTURE=$(TargetArchitecture) ICU_TRACING=$(IcuTracing) $(_ExtraParams) SHELL=/bin/bash"
+          Command="make -f icu.mk all TARGET_OS=$(TargetOS.ToLower()) TARGET_ARCHITECTURE=$(TargetArchitecture) ICU_TRACING=$(IcuTracing) ICU_SHARDING=$(IcuSharding) $(_ExtraParams) SHELL=/bin/bash"
           IgnoreStandardErrorWarningFormat="true" />
   </Target>
   <Target Name="Restore" />