[wasm] Enable ICU sharding

src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props

@@ -185,8 +185,24 @@
     <PlatformManifestFileEntry Include="icudt_no_CJK.dat" IsNative="true" />
     <PlatformManifestFileEntry Include="icudt_CJK.dat" IsNative="true" />
     <PlatformManifestFileEntry Include="icudt_EFIGS.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_efigs_locales.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_cjk_locales.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_no_cjk_locales.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_normalization.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_base.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_locales.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_currency.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_coll.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_efigs_coll.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_cjk_coll.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_no_cjk_coll.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_zones.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_coll.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_cjk_zones.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icudt_efigs_zones.dat" IsNative="true" />
     <PlatformManifestFileEntry Include="icudt_optimal.dat" IsNative="true" />
     <PlatformManifestFileEntry Include="icudt_optimal_no_CJK.dat" IsNative="true" />
+    <PlatformManifestFileEntry Include="icu_dictionary.json" IsNative="true" />
     <PlatformManifestFileEntry Include="binding_support.js" IsNative="true" />
     <PlatformManifestFileEntry Include="dotnet_support.js" IsNative="true" />
     <PlatformManifestFileEntry Include="library_mono.js" IsNative="true" />

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_static.c

@@ -47,7 +47,7 @@ static void U_CALLCONV icu_trace_data(const void* context, int32_t fnNumber, int
 #ifdef __EMSCRIPTEN__
 #include <emscripten.h>
 
-static int32_t load_icu_data(void* pData);
+static int32_t load_icu_data(void* pData, int32_t type);
 
 EMSCRIPTEN_KEEPALIVE const char* mono_wasm_get_icudt_name(const char* culture);
 
@@ -56,11 +56,11 @@ EMSCRIPTEN_KEEPALIVE const char* mono_wasm_get_icudt_name(const char* culture)
     return GlobalizationNative_GetICUDTName(culture);
 }
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData);
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData, int32_t type);
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData)
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData, int32_t type)
 {
-    return load_icu_data(pData);
+    return load_icu_data(pData, type);
 }
 
 
@@ -77,14 +77,22 @@ void mono_wasm_link_icu_shim(void)
 
 #endif
 
-static int32_t load_icu_data(void* pData)
+static int32_t load_icu_data(void* pData, int32_t type)
 {
 
     UErrorCode status = 0;
-    udata_setCommonData(pData, &status);
+    if (type == 0) {
+        udata_setAppData(NULL, pData, &status);
+    }
+    if (type == 1) {
+        udata_setCommonData(pData, &status);
+    }
 
     if (U_FAILURE(status)) {
-        log_icu_error("udata_setCommonData", status);
+        if (type)
+            log_icu_error("udata_setCommonData", status);
+        else
+            log_icu_error("udata_setAppData", status);
         return 0;
     } else {
 
@@ -146,7 +154,7 @@ int32_t GlobalizationNative_LoadICUData(const char* path)
 
     fclose(fp);
 
-    if (load_icu_data(icu_data) == 0) {
+    if (load_icu_data(icu_data, strcasecmp("icudt.dat", path)) == 0) {
         log_shim_error("ICU BAD EXIT %d.", ret);
         return ret;
     }

src/mono/sample/wasm/console/Makefile

@@ -1,4 +1,5 @@
 TOP=../../../../..
+CULTURE=en-us
 
 include ../wasm.mk
 
@@ -10,6 +11,10 @@ ifneq ($(V),)
 DOTNET_MONO_LOG_LEVEL=--setenv=MONO_LOG_LEVEL=debug
 endif
 
+ifneq ($(SHARD),)
+override MSBUILD_ARGS+=/p:EnableSharding=true /p:ICUDefaultCulture=$(CULTURE)
+endif
+
 PROJECT_NAME=Wasm.Console.Sample.csproj
 
 run: run-console

src/mono/sample/wasm/wasm.mk

@@ -23,7 +23,7 @@ run-browser:
 		echo "The tool dotnet-serve could not be found. Install with: $(DOTNET) tool install --global dotnet-serve"; \
 		exit 1; \
 	else  \
-		$(DOTNET) serve -d bin/$(CONFIG)/AppBundle -p 8000; \
+		$(DOTNET) serve --directory ./bin/$(CONFIG)/AppBundle -p 8000; \
 	fi
 
 run-console:

src/mono/wasm/build/WasmApp.targets

@@ -351,6 +351,7 @@
 
     <PropertyGroup>
       <WasmBuildNative Condition="'$(WasmBuildNative)' == 'true' and '$(_IsEMSDKMissing)' == 'true'">false</WasmBuildNative>
+      <ICUDefaultCulture Condition="'$(EnableSharding)' == 'true' and '$(WasmBuildNative)' == ''" >en-us</ICUDefaultCulture>
     </PropertyGroup>
   </Target>
 
@@ -360,29 +361,33 @@
     <Error Condition="'$(WasmMainJSPath)' == ''" Text="%24(WasmMainJSPath) property needs to be set" />
 
     <PropertyGroup>
-      <WasmIcuDataFileName Condition="'$(InvariantGlobalization)' != 'true'">icudt.dat</WasmIcuDataFileName>
-
       <_HasDotnetWasm Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.wasm'">true</_HasDotnetWasm>
       <_HasDotnetJs Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js'">true</_HasDotnetJs>
+      <IcuDictionaryPath Condition="'$(InvariantGlobalization)' != 'true'">$(MicrosoftNetCoreAppRuntimePackRidNativeDir)icu_dictionary.json</IcuDictionaryPath>
     </PropertyGroup>
 
     <ItemGroup>
       <!-- If dotnet.{wasm,js} weren't added already (eg. AOT can add them), then add the default ones -->
       <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.wasm" Condition="'$(_HasDotnetWasm)' != 'true'" />
       <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.js" Condition="'$(_HasDotnetJs)' != 'true'" />
 
-      <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)$(WasmIcuDataFileName)" Condition="'$(InvariantGlobalization)' != 'true'" />
+      <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)*.dat" Condition="'$(InvariantGlobalization)' != 'true'" />
       <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.timezones.blat" />
     </ItemGroup>
 
+    <ItemGroup Condition="'$(InvariantGlobalization)' != 'true'">
+      <WasmExtraConfig Include="enable_sharding" Value="true" Condition="'$(EnableSharding)' == 'true'" />
+      <WasmExtraConfig Include="default_culture" Value="&quot;$(ICUDefaultCulture)&quot;" Condition="'$(EnableSharding)' == 'true'" />
+    </ItemGroup>
+
     <WasmAppBuilder
       AppDir="$(WasmAppDir)"
       MainJS="$(WasmMainJSPath)"
       Assemblies="@(_WasmAssembliesInternal)"
       InvariantGlobalization="$(InvariantGlobalization)"
       SatelliteAssemblies="@(WasmSatelliteAssemblies)"
       FilesToIncludeInFileSystem="@(WasmFilesToIncludeInFileSystem)"
-      IcuDataFileName="$(WasmIcuDataFileName)"
+      IcuDictionary="$(IcuDictionaryPath)"
       RemoteSources="@(WasmRemoteSources)"
       ExtraFilesToDeploy="@(WasmExtraFilesToDeploy)"
       ExtraConfig="@(WasmExtraConfig)"

src/mono/wasm/runtime-test.js

@@ -2,7 +2,6 @@
 //
 // Run runtime tests under a JS shell or a browser
 //
-
 //glue code to deal with the differences between chrome, ch, d8, jsc and sm.
 var is_browser = typeof window != "undefined";
 
@@ -279,7 +278,13 @@ var Module = {
 				})
 			}
 		};
-
+		if (config.enable_sharding) {
+			if (config.default_culture != null) {
+				config.application_culture = config.default_culture;
+			} else {
+				config.application_culture = Intl.DateTimeFormat().resolvedOptions().locale;
+			}
+		}
 		MONO.mono_load_runtime_and_bcl_args (config);
 	},
 };

src/mono/wasm/runtime/library_mono.js

@@ -1135,6 +1135,47 @@ var MonoSupportLib = {
 			return res;
 		},
 
+		_get_shard_name: function (fileList) {
+			// Assumes that all filter files follow the naming convention:
+			// 		icudt_<feature>_<locale division>.json
+			// The more descriptors in a file name (separated by _) the granular the shard data is.
+
+			// Get a list of file name descriptor length 
+			var name_lengths = fileList.map(x => x.split("_").length);
+
+			// Choose the file with the most number of descriptors 
+			// Assuming that we want the most condensed version available
+			var shard_ix = name_lengths.indexOf(Math.max(...name_lengths));
+			return fileList[shard_ix];
+		},
+
+		_get_list_of_icu_files: function (dictionary, culture, feature_shards=true) {
+			var icu_files = []
+			if (culture == null || culture.length < 2) {
+				icu_files = [dictionary.complete];
+			} else {
+				var parent_culture = culture.includes('-') ? culture.split('-')[0] : culture;
+				console.log(parent_culture)
+				var files = dictionary[parent_culture];
+				if (!feature_shards) {
+					icu_files = [files.full];
+				} else {
+					icu_files.push(...files.essentials.reverse());
+					icu_files.push(this._get_shard_name(files.locales));
+					icu_files.push(this._get_shard_name(files.coll));
+				}
+			}
+
+			icu_assets = [];
+			icu_files.forEach(file => icu_assets.push({
+																									"behavior": "icu",
+																									"name": file,
+																									"load_remote": false,
+																									"data_type": "common"
+																								}));
+			return icu_assets;
+		},
+
 		mono_wasm_get_details: function (objectId, args={}) {
 			let id = this._parse_object_id (objectId, true);
 
@@ -1524,7 +1565,6 @@ var MonoSupportLib = {
 
 			var virtualName = asset.virtual_path || asset.name;
 			var offset = null;
-
 			switch (asset.behavior) {
 				case "resource":
 				case "assembly":
@@ -1581,7 +1621,7 @@ var MonoSupportLib = {
 				}
 			}
 			else if (asset.behavior === "icu") {
-				if (this.mono_wasm_load_icu_data (offset))
+				if (this.mono_wasm_load_icu_data (offset, +(asset.data_type == "common")))
 					ctx.num_icu_assets_loaded_successfully += 1;
 				else
 					console.error ("Error loading ICU asset", asset.name);
@@ -1660,6 +1700,7 @@ var MonoSupportLib = {
 		//      "icu": load ICU globalization data from any runtime assets with behavior "icu".
 		//      "invariant": operate in invariant globalization mode.
 		//      "auto" (default): if "icu" behavior assets are present, use ICU, otherwise invariant.
+		//		application_culture: (optional) current browser culture
 		//    diagnostic_tracing: (optional) enables diagnostic log messages during startup
 		mono_load_runtime_and_bcl_args: function (args) {
 			try {
@@ -1683,9 +1724,9 @@ var MonoSupportLib = {
 
 		// @offset must be the address of an ICU data archive in the native heap.
 		// returns true on success.
-		mono_wasm_load_icu_data: function (offset) {
-			var fn = Module.cwrap ('mono_wasm_load_icu_data', 'number', ['number']);
-			var ok = (fn (offset)) === 1;
+		mono_wasm_load_icu_data: function (offset, type) {
+			var fn = Module.cwrap ('mono_wasm_load_icu_data', 'number', ['number', 'number']);
+			var ok = (fn (offset, type)) === 1;
 			if (ok)
 				this.num_icu_assets_loaded_successfully++;
 			return ok;
@@ -1751,7 +1792,8 @@ var MonoSupportLib = {
 				throw new Error ("Invalid args (runtime_asset_sources was replaced by remote_sources)");
 			if (!args.loaded_cb)
 				throw new Error ("loaded_cb not provided");
-
+
+			args.assets = args.assets.concat(this._get_list_of_icu_files(args.icu_dictionary, args.application_culture));
 			var ctx = {
 				tracing: args.diagnostic_tracing || false,
 				pending_count: args.assets.length,
@@ -1799,7 +1841,6 @@ var MonoSupportLib = {
 				var attemptNextSource;
 				var sourceIndex = 0;
 				var sourcesList = asset.load_remote ? args.remote_sources : [""];
-
 				var handleFetchResponse = function (response) {
 					if (!response.ok) {
 						try {

src/tasks/WasmAppBuilder/WasmAppBuilder.cs

@@ -41,7 +41,7 @@ public class WasmAppBuilder : Task
 
     // full list of ICU data files we produce can be found here:
     // https://github.com/dotnet/icu/tree/maint/maint-67/icu-filters
-    public string? IcuDataFileName { get; set; }
+    public string? IcuDictionary { get; set; }
 
     public int DebugLevel { get; set; }
     public ITaskItem[]? SatelliteAssemblies { get; set; }
@@ -119,14 +119,21 @@ private class IcuData : AssetEntry
         public IcuData(string name) : base(name, "icu") {}
         [JsonPropertyName("load_remote")]
         public bool LoadRemote { get; set; }
+        [JsonPropertyName("data_type")]
+        public string? DataType { get; set; }
     }
 
     public override bool Execute ()
     {
-        if (!File.Exists(MainJS))
-            throw new ArgumentException($"File MainJS='{MainJS}' doesn't exist.");
-        if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDataFileName))
-            throw new ArgumentException("IcuDataFileName property shouldn't be empty if InvariantGlobalization=false");
+        if (!File.Exists(MainJS)) {
+            Log.LogError($"File MainJS='{MainJS}' doesn't exist.");
+            return false;
+        }
+        if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDictionary))
+        {
+            Log.LogError("IcuDictionary property shouldn't be empty if InvariantGlobalization=false");
+            return false;
+        }
 
         if (Assemblies?.Length == 0)
         {
@@ -225,9 +232,6 @@ public override bool Execute ()
             }
         }
 
-        if (!InvariantGlobalization)
-            config.Assets.Add(new IcuData(IcuDataFileName!) { LoadRemote = RemoteSources?.Length > 0 });
-
         config.Assets.Add(new VfsEntry ("dotnet.timezones.blat") { VirtualPath = "/usr/share/zoneinfo/"});
 
         if (RemoteSources?.Length > 0)
@@ -246,6 +250,20 @@ public override bool Execute ()
             config.Extra[name] = valueObject;
         }
 
+        if (!InvariantGlobalization)
+        {
+            try
+            {
+                string? icuDictionary = File.ReadAllText(IcuDictionary!);
+                config.Extra["icu_dictionary"] = JsonSerializer.Deserialize<Dictionary<string, object>>(icuDictionary!);
+            }
+            catch (Exception e)
+            {
+                Log.LogError($"Error with opening ICU Dictionary {e.Message}");
+                return false;
+            }
+        }
+
         string monoConfigPath = Path.Join(AppDir, "mono-config.js");
         using (var sw = File.CreateText(monoConfigPath))
         {