Skip to content

Commit

Permalink
Applied changes from old ICU PR: dotnet#104.
Browse files Browse the repository at this point in the history
  • Loading branch information
ilonatommy committed Jul 19, 2022
1 parent 36c0bf2 commit f7b8e5c
Show file tree
Hide file tree
Showing 27 changed files with 5,501 additions and 36 deletions.
48 changes: 48 additions & 0 deletions SHARDS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# Building ICU Shards

The ICU is divided by locales and features. <br/>

By locale:
- EFIGS (en, fr, it, de, es)
- CJK (zh, ja, ko)
- no CJK (all locales except for zh, ja, ko)

By features:
- Collation
- Normalization
- Currency
- Locales
- Zones

To generate ICU shards run:

`make -C ./eng -f icu.mk shards`

which will build all shards from filter files available in `icu-filters/` as well as `icu-dictionary.json`, which maps each parent locale (i.e. en) to relevant files.

## ICU dictionary
The ICU dictionary is divided into the following format:

```
{
"en": {
"essentials": [
icudt_currency.dat,
icudt_normalization.dat,
icudt_base.dat
]
"zones": [ relevant timezone data files ],
"locales": [ relevant locale data files ],
"coll": [ relevant collationd data files ]
}
.
.
.
}
```

To generate just the ICU dictionary run:

`make -C ./eng -f icu.mk icu_dictionary.json`

The dictionary is packaged with the data files to be consumed later by the WASM runtime.
4 changes: 2 additions & 2 deletions eng/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ stages:
############ BROWSER BUILD ############
- job: Build_Browser
displayName: Browser
timeoutInMinutes: 30
timeoutInMinutes: 40
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
vmImage: ubuntu-18.04
Expand All @@ -43,7 +43,7 @@ stages:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220531132048-00a561c
steps:
- bash: |
./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm $(_InternalBuildArgs)
./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:IcuSharding=true $(_InternalBuildArgs)
displayName: Build
- task: PublishPipelineArtifact@1
displayName: Upload artifacts
Expand Down
1 change: 1 addition & 0 deletions eng/icu.browser.mk
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
ENV_INIT_SCRIPT = source $(EMSDK_PATH)/emsdk_env.sh &&
ENV_CONFIGURE_WRAPPER = emconfigure
ICU_SHARDING = true

ifeq ($(WASM_ENABLE_THREADS),true)
THREADS_FLAG="-pthread"
Expand Down
34 changes: 33 additions & 1 deletion eng/icu.mk
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,38 @@ endif
$(eval $(call TargetBuildTemplate,icudt_CJK,icudt_CJK))
$(eval $(call TargetBuildTemplate,icudt_no_CJK,icudt_no_CJK))
$(eval $(call TargetBuildTemplate,icudt_EFIGS,icudt_EFIGS))
$(eval $(call TargetBuildTemplate,icudt_normalization,icudt_normalization))
$(eval $(call TargetBuildTemplate,icudt_base,icudt_base))
$(eval $(call TargetBuildTemplate,icudt_currency,icudt_currency))
$(eval $(call TargetBuildTemplate,icudt_full_full,icudt_full_full))
$(eval $(call TargetBuildTemplate,icudt_full_coll,icudt_full_coll))
$(eval $(call TargetBuildTemplate,icudt_full_zones,icudt_full_zones))
$(eval $(call TargetBuildTemplate,icudt_full_locales,icudt_full_locales))
$(eval $(call TargetBuildTemplate,icudt_efigs_locales,icudt_efigs_locales))
$(eval $(call TargetBuildTemplate,icudt_efigs_coll,icudt_efigs_coll))
$(eval $(call TargetBuildTemplate,icudt_efigs_full,icudt_efigs_full))
$(eval $(call TargetBuildTemplate,icudt_efigs_zones,icudt_efigs_zones))
$(eval $(call TargetBuildTemplate,icudt_cjk_locales,icudt_cjk_locales))
$(eval $(call TargetBuildTemplate,icudt_cjk_zones,icudt_cjk_zones))
$(eval $(call TargetBuildTemplate,icudt_cjk_full,icudt_cjk_full))
$(eval $(call TargetBuildTemplate,icudt_cjk_coll,icudt_cjk_coll))
$(eval $(call TargetBuildTemplate,icudt_no_cjk_full,icudt_no_cjk_full))
$(eval $(call TargetBuildTemplate,icudt_no_cjk_coll,icudt_no_cjk_coll))
$(eval $(call TargetBuildTemplate,icudt_no_cjk_zones,icudt_no_cjk_zones))
$(eval $(call TargetBuildTemplate,icudt_no_cjk_locales,icudt_no_cjk_locales))


ICU_SHARDS := icudt_base icudt_normalization icudt_currency icudt_cjk_zones icudt_no_cjk_zones icudt_efigs_zones icudt_efigs_locales icudt_cjk_locales icudt_no_cjk_locales icudt_efigs_coll icudt_cjk_coll icudt_no_cjk_coll icudt_efigs_full icudt_cjk_full icudt_no_cjk_full icudt_full_full
DATA_SHARDS := $(addprefix data-, $(ICU_SHARDS))

dictionary:
cd $(TOP)/icu/icu4c/source/ && PYTHONPATH=python python3 -m icutools.databuilder --mode=makedict --filter_dir=$(ICU_FILTER_PATH) --out_dir=$(TARGET_BINDIR) --shard_cfg=$(abspath $(CURDIR)/main-config.json) --exclude_feats=zones;

shards: dictionary $(DATA_SHARDS)

# build source+data for the main "icudt" filter and only data for the other filters
all: lib-icudt data-icudt data-icudt_no_CJK data-icudt_EFIGS data-icudt_CJK
ifeq ($(ICU_SHARDING), true)
all: lib-icudt data-icudt data-icudt_no_CJK data-icudt_EFIGS data-icudt_CJK dictionary shards
else
all: lib-icudt data-icudt data-icudt_no_CJK data-icudt_EFIGS data-icudt_CJK
endif
64 changes: 32 additions & 32 deletions eng/icu.proj
Original file line number Diff line number Diff line change
@@ -1,40 +1,40 @@
<Project>
<Target Name="Build" Condition="'$(PackOnly)' != 'true'" DependsOnTargets="PatchEmscripten;CopyEmscriptenVersion">
<Error Condition="'$(TargetOS)' == ''" Text="TargetOS needs to be specified!" />
<Error Condition="'$(TargetArchitecture)' == ''" Text="TargetArchitecture needs to be specified!" />
<Target Name="Build" Condition="'$(PackOnly)' != 'true'" DependsOnTargets="PatchEmscripten;CopyEmscriptenVersion">
<Error Condition="'$(TargetOS)' == ''" Text="TargetOS needs to be specified!" />
<Error Condition="'$(TargetArchitecture)' == ''" Text="TargetArchitecture needs to be specified!" />

<PropertyGroup>
<_ExtraParams Condition="'$(WasmEnableThreads)' == 'true'">WASM_ENABLE_THREADS=true</_ExtraParams>
</PropertyGroup>
<PropertyGroup>
<_ExtraParams Condition="'$(WasmEnableThreads)' == 'true'">WASM_ENABLE_THREADS=true</_ExtraParams>
</PropertyGroup>

<Copy SourceFiles="$(MSBuildThisFileDirectory)\..\icu-filters\curr-root.txt"
DestinationFiles="$(MSBuildThisFileDirectory)\..\icu\icu4c\source\data\curr\root.txt"/>
<Exec WorkingDirectory="$(MSBuildThisFileDirectory)"
Command="make -f icu.mk all TARGET_OS=$(TargetOS.ToLower()) TARGET_ARCHITECTURE=$(TargetArchitecture) ICU_TRACING=$(IcuTracing) $(_ExtraParams) SHELL=/bin/bash"
IgnoreStandardErrorWarningFormat="true" />
</Target>
<Target Name="Restore" />
<Target Name="Test" />
<Target Name="Pack" DependsOnTargets="Build">
<MSBuild Projects="$(MSBuildThisFileDirectory)nuget\Microsoft.NETCore.Runtime.ICU.Transport.pkgproj" Targets="Build" />
</Target>
<Copy SourceFiles="$(MSBuildThisFileDirectory)\..\icu-filters\curr-root.txt"
DestinationFiles="$(MSBuildThisFileDirectory)\..\icu\icu4c\source\data\curr\root.txt"/>
<Exec WorkingDirectory="$(MSBuildThisFileDirectory)"
Command="make -f icu.mk all TARGET_OS=$(TargetOS.ToLower()) TARGET_ARCHITECTURE=$(TargetArchitecture) ICU_TRACING=$(IcuTracing) ICU_SHARDING=$(IcuSharding) $(_ExtraParams) SHELL=/bin/bash"
IgnoreStandardErrorWarningFormat="true" />
</Target>
<Target Name="Restore" />
<Target Name="Test" />
<Target Name="Pack" DependsOnTargets="Build">
<MSBuild Projects="$(MSBuildThisFileDirectory)nuget\Microsoft.NETCore.Runtime.ICU.Transport.pkgproj" Targets="Build" />
</Target>

<!--
<!--
The version of emscripten we are on, emcmake does not pass 'experimental-wasm-threads' to node. In order for the build to
succeed, we need to provide a patched version of that file. Ideally, fixing upstream would be best, but until then, patch.
-->
<Target Name="PatchEmscripten" Condition="'$(TargetOS)' == 'Browser' and '$(WasmEnableThreads)' == 'true'">
<Copy SourceFiles="$(MSBuildThisFileDirectory)patches\emcc.py"
DestinationFolder="$(EMSDK_PATH)\upstream\emscripten" />
</Target>
<Target Name="CopyEmscriptenVersion" Condition="'$(TargetOS)' == 'Browser'">
<ReadLinesFromFile File="$(EMSDK_PATH)\upstream\emscripten\emscripten-version.txt" >
<Output TaskParameter="Lines"
ItemName="_ItemsFromEmscriptenVersionFile"/>
</ReadLinesFromFile>
<WriteLinesToFile
File="$(RepoRoot)\artifacts\bin\icu-browser-wasm\emscripten-version.txt"
Lines="@(_ItemsFromEmscriptenVersionFile->Replace('&quot;', ''))"
Overwrite="true" />
</Target>
<Target Name="PatchEmscripten" Condition="'$(TargetOS)' == 'Browser' and '$(WasmEnableThreads)' == 'true'">
<Copy SourceFiles="$(MSBuildThisFileDirectory)patches\emcc.py"
DestinationFolder="$(EMSDK_PATH)\upstream\emscripten" />
</Target>
<Target Name="CopyEmscriptenVersion" Condition="'$(TargetOS)' == 'Browser'">
<ReadLinesFromFile File="$(EMSDK_PATH)\upstream\emscripten\emscripten-version.txt" >
<Output TaskParameter="Lines"
ItemName="_ItemsFromEmscriptenVersionFile"/>
</ReadLinesFromFile>
<WriteLinesToFile
File="$(RepoRoot)\artifacts\bin\icu-browser-wasm\emscripten-version.txt"
Lines="@(_ItemsFromEmscriptenVersionFile->Replace('&quot;', ''))"
Overwrite="true" />
</Target>
</Project>
Loading

0 comments on commit f7b8e5c

Please sign in to comment.