Merge pull request #146 from Baptistemontan/build

crate for build.rs utilities

.github/workflows/ci.yml

@@ -35,12 +35,12 @@ jobs:
     # so if some features mix creates a problem they will appear in leptos_i18n tests.
     # All leptos_i18n_macro tests don't require any feature flag, so no need to test each feature and matchup
     # just run once with default, this speeds up CI
-    name: Test leptos_i18n_macro package
+    name: Test ${{ matrix.packages }} package
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        packages: [leptos_i18n_macro, leptos_i18n_parser]
+        packages: [leptos_i18n_macro, leptos_i18n_parser, leptos_i18n_build]
     steps:
       - name: "Checkout repo"
         uses: actions/checkout@v4

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "leptos_i18n",
     "leptos_i18n_macro",
     "leptos_i18n_parser",
+    "leptos_i18n_build",
     "tests/json",
     "tests/common",
     "tests/namespaces",

docs/book/src/reduce_size/01_datagen.md

@@ -101,6 +101,74 @@ fn main() {
 
 Here we are generating the informations for locales `"en"` and `"fr"`, with the data needed for plurals.
 
+## Using `leptos_i18n_build` crate
+
+You can use the `leptos_i18n_build` crate that contains utils for the datagen.
+The problem with above `build.rs` is that it can go out of sync with your translations,
+when all informations are already in the translations.
+
+```toml
+# Cargo.toml
+[build-dependencies]
+leptos_i18n_build = "0.5.0-gamma"
+```
+
+```rust
+use leptos_i18n_build::TranslationsInfos;
+use std::path::PathBuf;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=Cargo.toml");
+
+    let mod_directory = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("baked_data");
+
+    let translations_infos = TranslationsInfos::parse().unwrap();
+
+    translations_infos.rerun_if_locales_changed();
+
+    translations_infos.generate_data(mod_directory).unwrap();
+}
+```
+
+This will parse the config and the translations and generate the data for you using the informations gained when parsing the translations.
+This will trigger a rerun if the config or translations changed and be kept in sync.
+If your code use plurals, it will build with informations for plurals. If it use a formatter it will build with the informations for that formatter.
+
+If you use more data someway, like for example using `t*_format!` with a formatter not used in the translations, there is functions to either supply additionnal options or keys:
+
+```rust
+use leptos_i18n_build::Options;
+
+translations_infos.generate_data_with_options(mod_directory, [Options::FormatDateTime]).unwrap();
+```
+
+This will inject the ICU `DataKey`s needed for the `date`, `time` and `datetime` formatters.
+
+```rust
+use leptos_i18n_build::Options;
+
+translations_infos.generate_data_with_data_keys(
+    mod_directory,
+    icu_datagen::keys(&["plurals/cardinal@1", "plurals/ordinal@1"])
+).unwrap();
+```
+
+This will inject the keys for cardinal and ordinal plurals.
+
+If you need both, `Options` can be turned into the need keys:
+
+```rust
+use leptos_i18n_build::Options;
+
+let mut keys = icu_datagen::keys(&["plurals/cardinal@1", "plurals/ordinal@1"])
+let keys.extend(Options::FormatDateTime.into_data_keys())
+
+// keys now contains the `DataKey`s needed for plurals and for the `time`, `date` and `datetime` formatters.
+
+translations_infos.generate_data_with_data_keys(mod_directory, keys).unwrap();
+```
+
 ## Is it worth the trouble ?
 
 YES. With `opt-level = "z"` and `lto = true`, the plurals example is at 394ko (at the time of writing), now by just providing a custom provider tailored to the used locales ("en" and "fr"), it shrinks down to 248ko! It almost cutted in half the binary size!

examples/csr/counter_icu_datagen/Cargo.toml

@@ -24,8 +24,7 @@ default = "en"
 locales = ["en", "fr"]
 
 [build-dependencies]
-icu = "1.5"
-icu_datagen = "1.5"
+leptos_i18n_build = { path = "../../../leptos_i18n_build" }
 
 [profile.release]
 opt-level = "z"

examples/csr/counter_icu_datagen/build.rs

@@ -1,28 +1,15 @@
-use icu_datagen::baked_exporter::*;
-use icu_datagen::prelude::*;
+use leptos_i18n_build::TranslationsInfos;
 use std::path::PathBuf;
 
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=Cargo.toml");
 
     let mod_directory = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("baked_data");
 
-    // This is'nt really needed, but ICU4X wants the directory to be empty
-    // and Rust Analyzer can trigger the build.rs without cleaning the out directory.
-    if mod_directory.exists() {
-        std::fs::remove_dir_all(&mod_directory).unwrap();
-    }
+    let translations_infos = TranslationsInfos::parse().unwrap();
 
-    let exporter = BakedExporter::new(mod_directory, Default::default()).unwrap();
+    translations_infos.rerun_if_locales_changed();
 
-    DatagenDriver::new()
-        // Keys needed for plurals
-        .with_keys(icu_datagen::keys(&[
-            "plurals/cardinal@1",
-            "plurals/ordinal@1",
-        ]))
-        // Used locales, no fallback needed
-        .with_locales_no_fallback([langid!("en"), langid!("fr")], Default::default())
-        .export(&DatagenProvider::new_latest_tested(), exporter)
-        .unwrap();
+    translations_infos.generate_data(mod_directory).unwrap();
 }

leptos_i18n/src/lib.rs

@@ -150,15 +150,6 @@ pub mod __private {
 }
 
 /// This module contain utilities to create custom ICU providers.
-#[cfg(all(
-    not(feature = "icu_compiled_data"),
-    any(
-        feature = "format_nums",
-        feature = "format_datetime",
-        feature = "format_list",
-        feature = "plurals"
-    )
-))]
 pub mod custom_provider {
     pub use crate::macro_helpers::formatting::data_provider::IcuDataProvider;
     pub use crate::macro_helpers::formatting::inner::set_icu_data_provider;

leptos_i18n/src/macro_helpers/formatting/mod.rs

@@ -222,24 +222,48 @@ pub(crate) mod inner {
         pub provider: data_provider::BakedDataProvider,
     }
 
-    #[cfg(not(feature = "icu_compiled_data"))]
     /// Supply a custom ICU data provider
+    /// Does nothing if the "icu_compiled_data" feature is enabled.
     pub fn set_icu_data_provider(data_provider: impl super::data_provider::IcuDataProvider) {
+        #[cfg(feature = "icu_compiled_data")]
+        let _ = data_provider;
+        #[cfg(not(feature = "icu_compiled_data"))]
         inner::FORMATTERS.with_mut(|formatters| {
             formatters.provider =
                 super::data_provider::BakedDataProvider(Some(Box::new(data_provider)));
         });
     }
 }
 
-#[cfg(any(
+#[cfg(not(any(
     feature = "format_nums",
     feature = "format_datetime",
     feature = "format_list",
     feature = "plurals"
-))]
+)))]
+pub(crate) mod inner {
+    /// Supply a custom ICU data provider
+    /// Does nothing if the "icu_compiled_data" feature is enabled.
+    pub fn set_icu_data_provider(data_provider: impl super::data_provider::IcuDataProvider) {
+        let _ = data_provider;
+    }
+}
+
 pub(crate) mod data_provider {
+    #[cfg(any(
+        feature = "format_nums",
+        feature = "format_datetime",
+        feature = "format_list",
+        feature = "plurals"
+    ))]
     use super::*;
+
+    #[cfg(any(
+        feature = "format_nums",
+        feature = "format_datetime",
+        feature = "format_list",
+        feature = "plurals"
+    ))]
     use icu_provider::DataLocale;
 
     /// Trait for custom ICU data providers.
@@ -394,6 +418,7 @@ pub(crate) mod data_provider {
 
     #[cfg(not(feature = "icu_compiled_data"))]
     impl BakedDataProvider {
+        #[allow(dead_code)]
         fn get_provider(&self) -> &dyn IcuDataProvider {
             self.0.as_deref().expect("No DataProvider provided.")
         }

leptos_i18n_build/.gitignore

@@ -0,0 +1,2 @@
+/target
+/Cargo.lock

leptos_i18n_build/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "leptos_i18n_build"
+version = { workspace = true }
+edition = "2021"
+authors = ["Baptiste de Montangon"]
+license = "MIT"
+repository = "https://github.com/Baptistemontan/leptos_i18n"
+description = "build.rs utilities for the leptos_i18n crate"
+readme = "../README.md"
+
+[dependencies]
+leptos_i18n_parser = { workspace = true }
+icu_datagen = { version = "1.5" }
+icu = { version = "1.5" }
+icu_provider = { version = "1.5" }
+
+[features]
+default = ["json_files"]
+json_files = ["leptos_i18n_parser/json_files"]
+yaml_files = ["leptos_i18n_parser/yaml_files"]
+json5_files = ["leptos_i18n_parser/json5_files"]

leptos_i18n_build/src/datakey.rs

@@ -0,0 +1,103 @@
+use std::collections::HashSet;
+
+use icu_datagen::prelude::DataKey;
+use leptos_i18n_parser::{
+    parse_locales::locale::{BuildersKeysInner, InterpolOrLit, LocaleValue, RangeOrPlural},
+    utils::formatter::Formatter,
+};
+
+/// This enum represent the different `Fromatters` and options your translations could be using.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Options {
+    /// Use of plurals.
+    Plurals,
+    /// Use of the `date`, `time` or `datetime` formatter.
+    FormatDateTime,
+    /// Use of the `list` formatter.
+    FormatList,
+    /// Use of the `number` formatter.
+    FormatNums,
+}
+
+pub fn find_used_datakey(keys: &BuildersKeysInner, used_icu_keys: &mut HashSet<Options>) {
+    for locale_value in keys.0.values() {
+        match locale_value {
+            LocaleValue::Subkeys { keys, .. } => find_used_datakey(keys, used_icu_keys),
+            LocaleValue::Value(InterpolOrLit::Lit(_)) => {} // skip literals
+            LocaleValue::Value(InterpolOrLit::Interpol(interpolation_keys)) => {
+                for (_, var_infos) in interpolation_keys.iter_vars() {
+                    if matches!(var_infos.range_count, Some(RangeOrPlural::Plural)) {
+                        used_icu_keys.insert(Options::Plurals);
+                    }
+
+                    for formatter in &var_infos.formatters {
+                        let dk = match formatter {
+                            Formatter::None => continue,
+                            Formatter::Number => Options::FormatNums,
+                            Formatter::Date(_) | Formatter::Time(_) | Formatter::DateTime(_, _) => {
+                                Options::FormatDateTime
+                            }
+                            Formatter::List(_, _) => Options::FormatList,
+                        };
+                        used_icu_keys.insert(dk);
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub fn get_keys(used_icu_keys: impl IntoIterator<Item = Options>) -> impl Iterator<Item = DataKey> {
+    used_icu_keys.into_iter().flat_map(Options::into_data_keys)
+}
+
+impl Options {
+    /// Return a `Vec<DataKey>` needed to use the given option.
+    pub fn into_data_keys(self) -> Vec<DataKey> {
+        match self {
+            Options::Plurals => icu_datagen::keys(&["plurals/cardinal@1", "plurals/ordinal@1"]),
+            Options::FormatDateTime => icu_datagen::keys(&[
+                "datetime/timesymbols@1",
+                "datetime/timelengths@1",
+                "datetime/skeletons@1",
+                "plurals/ordinal@1",
+                "datetime/week_data@1",
+                "decimal/symbols@1",
+                "datetime/gregory/datelengths@1",
+                "datetime/gregory/datesymbols@1",
+                "datetime/buddhist/datelengths@1",
+                "datetime/buddhist/datesymbols@1",
+                "calendar/chinesecache@1",
+                "datetime/chinese/datelengths@1",
+                "datetime/chinese/datesymbols@1",
+                "datetime/coptic/datelengths@1",
+                "datetime/coptic/datesymbols@1",
+                "calendar/dangicache@1",
+                "datetime/dangi/datelengths@1",
+                "datetime/dangi/datesymbols@1",
+                "datetime/ethiopic/datelengths@1",
+                "datetime/ethiopic/datesymbols@1",
+                "datetime/hebrew/datelengths@1",
+                "datetime/hebrew/datesymbols@1",
+                "datetime/indian/datelengths@1",
+                "datetime/indian/datesymbols@1",
+                "datetime/islamic/datelengths@1",
+                "datetime/islamic/datesymbols@1",
+                "calendar/islamicobservationalcache@1",
+                "calendar/islamicummalquracache@1",
+                "datetime/japanese/datelengths@1",
+                "datetime/japanese/datesymbols@1",
+                "calendar/japanese@1",
+                "datetime/japanext/datelengths@1",
+                "datetime/japanext/datesymbols@1",
+                "calendar/japanext@1",
+                "datetime/persian/datelengths@1",
+                "datetime/persian/datesymbols@1",
+                "datetime/roc/datelengths@1",
+                "datetime/roc/datesymbols@1",
+            ]),
+            Options::FormatList => icu_datagen::keys(&["list/and@1", "list/or@1", "list/unit@1"]),
+            Options::FormatNums => icu_datagen::keys(&["decimal/symbols@1"]),
+        }
+    }
+}