diff --git a/Cargo.toml b/Cargo.toml index 5f5abd7..da04644 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,17 @@ categories = ["localization", "internationalization"] description = "Rust I18n is use Rust codegen for load YAML file storage translations on compile time, and give you a t! macro for simply get translation texts." edition = "2021" exclude = ["crates", "tests"] -keywords = ["i18n", "yml", "localization", "internationalization"] +keywords = [ + "gettext", + "i18n", + "l10n", + "intl", + "internationalization", + "localization", + "tr", + "translation", + "yml", +] license = "MIT" name = "rust-i18n" readme = "README.md" @@ -28,6 +38,9 @@ serde_yaml = "0.8" globwalk = "0.8.1" regex = "1" +[features] +log-miss-tr = ["rust-i18n-macro/log-miss-tr"] + [[example]] name = "app" test = true @@ -38,10 +51,17 @@ members = [ "crates/extract", "crates/support", "crates/macro", + "examples/app-egui", "examples/app-load-path", + "examples/app-metadata", + "examples/app-minify-key", "examples/foo", ] [[bench]] harness = false name = "bench" + +[[bench]] +harness = false +name = "minify_key" diff --git a/README.md b/README.md index b8517bb..2af2c22 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ > 🎯 Let's make I18n things to easy! -Rust I18n is a crate for loading localized text from a set of (YAML, JSON or TOML) mapping files. The mappings are converted into data readable by Rust programs at compile time, and then localized text can be loaded by simply calling the provided `t!` macro. +Rust I18n is a crate for loading localized text from a set of (YAML, JSON or TOML) mapping files. The mappings are converted into data readable by Rust programs at compile time, and then localized text can be loaded by simply calling the provided [`t!`] macro. Unlike other I18n libraries, Rust I18n's goal is to provide a simple and easy-to-use API. @@ -13,10 +13,15 @@ The API of this crate is inspired by [ruby-i18n](https://github.com/ruby-i18n/i1 ## Features - Codegen on compile time for includes translations into binary. -- Global `t!` macro for loading localized text in everywhere. +- Global [`t!`] macro for loading localized text in everywhere. - Use YAML (default), JSON or TOML format for mapping localized text, and support mutiple files merging. - `cargo i18n` Command line tool for checking and extract untranslated texts into YAML files. - Support all localized texts in one file, or split into difference files by locale. +- Supports specifying a chain of fallback locales for missing translations. +- Supports automatic lookup of language territory for fallback locale. For instance, if `zh-CN` is not available, it will fallback to `zh`. (Since v2.4.0) +- Support short hashed keys for optimize memory usage and lookup speed. (Since v3.1.0) +- Support format variables in [`t!`], and support format variables with [`std::fmt`](https://doc.rust-lang.org/std/fmt/) syntax. (Since v3.1.0) +- Support for log missing translations at the warning level with `log-miss-tr` feature, the feature requires the `log` crate. (Since v3.1.0) ## Usage @@ -39,16 +44,36 @@ i18n!("locales"); // Or just use `i18n!`, default locales path is: "locales" in current crate. // -// i18n!(); +i18n!(); // Config fallback missing translations to "en" locale. // Use `fallback` option to set fallback locale. // -// i18n!("locales", fallback = "en"); +i18n!("locales", fallback = "en"); // Or more than one fallback with priority. // -// i18n!("locales", fallback = ["en", "es]); +i18n!("locales", fallback = ["en", "es"]); + +// Use a short hashed key as an identifier for long string literals +// to optimize memory usage and lookup speed. +// The key generation algorithm is `${Prefix}${Base62(SipHash13("msg"))}`. +i18n!("locales", minify_key = true); +// +// Alternatively, you can customize the key length, prefix, +// and threshold for the short hashed key. +i18n!("locales", + minify_key = true, + minify_key_len = 12, + minify_key_prefix = "T.", + minify_key_thresh = 64 +); +// Now, if the message length exceeds 64, the `t!` macro will automatically generate +// a 12-byte short hashed key with a "T." prefix for it, if not, it will use the original. + +// Configuration using the `[package.metadata.i18n]` section in `Cargo.toml`, +// Useful for the `cargo i18n` command line tool. +i18n!(metadata = true); ``` Or you can import by use directly: @@ -60,6 +85,7 @@ use rust_i18n::t; rust_i18n::i18n!("locales"); fn main() { + // Find the translation for the string literal `Hello` using the manually provided key `hello`. println!("{}", t!("hello")); // Use `available_locales!` method to get all available locales. @@ -90,8 +116,11 @@ You can also split the each language into difference files, and you can choise ( ```yml _version: 1 -hello: 'Hello world' -messages.hello: 'Hello, %{name}' +hello: "Hello world" +messages.hello: "Hello, %{name}" + +# Generate short hashed keys using `minify_key=true, minify_key_thresh=10` +4Cct6Q289b12SkvF47dXIx: "Hello, %{name}" ``` Or use JSON or TOML format, just rename the file to `en.json` or `en.toml`, and the content is like this: @@ -100,13 +129,19 @@ Or use JSON or TOML format, just rename the file to `en.json` or `en.toml`, and { "_version": 1, "hello": "Hello world", - "messages.hello": "Hello, %{name}" + "messages.hello": "Hello, %{name}", + + // Generate short hashed keys using `minify_key=true, minify_key_thresh=10` + "4Cct6Q289b12SkvF47dXIx": "Hello, %{name}" } ``` ```toml hello = "Hello world" +# Generate short hashed keys using `minify_key=true, minify_key_thresh=10` +4Cct6Q289b12SkvF47dXIx = "Hello, %{name}" + [messages] hello = "Hello, %{name}" ``` @@ -144,6 +179,11 @@ hello: messages.hello: en: Hello, %{name} zh-CN: 你好，%{name} + +# Generate short hashed keys using `minify_key=true, minify_key_thresh=10` +4Cct6Q289b12SkvF47dXIx: + en: Hello, %{name} + zh-CN: 你好，%{name} ``` This is useful when you use [GitHub Copilot](https://github.com/features/copilot), after you write a first translated text, then Copilot will auto generate other locale's translations for you. @@ -152,7 +192,7 @@ This is useful when you use [GitHub Copilot](https://github.com/features/copilot ### Get Localized Strings in Rust -Import the `t!` macro from this crate into your current scope: +Import the [`t!`] macro from this crate into your current scope: ```rust,no_run use rust_i18n::t; @@ -161,9 +201,11 @@ use rust_i18n::t; Then, simply use it wherever a localized string is needed: ```rust,no_run -# fn _rust_i18n_translate(locale: &str, key: &str) -> String { todo!() } +# macro_rules! t { +# ($($all_tokens:tt)*) => {} +# } # fn main() { -use rust_i18n::t; +// use rust_i18n::t; t!("hello"); // => "Hello world" @@ -181,12 +223,15 @@ t!("messages.hello", locale = "zh-CN", name = "Jason", count = 2); t!("messages.hello", locale = "zh-CN", "name" => "Jason", "count" => 3 + 2); // => "你好，Jason (5)" + +t!("Hello, %{name}, you serial number is: %{sn}", name = "Jason", sn = 123 : {:08}); +// => "Hello, Jason, you serial number is: 000000123" # } ``` ### Current Locale -You can use `rust_i18n::set_locale` to set the global locale at runtime, so that you don't have to specify the locale on each `t!` invocation. +You can use [`rust_i18n::set_locale()`]() to set the global locale at runtime, so that you don't have to specify the locale on each [`t!`] invocation. ```rust rust_i18n::set_locale("zh-CN"); @@ -259,7 +304,7 @@ rust_i18n::i18n!("locales", backend = RemoteI18n::new()); This also will load local translates from ./locales path, but your own `RemoteI18n` will priority than it. -Now you call `t!` will lookup translates from your own backend first, if not found, will lookup from local files. +Now you call [`t!`] will lookup translates from your own backend first, if not found, will lookup from local files. ## Example @@ -341,24 +386,39 @@ Run `cargo i18n -h` to see details. ```bash $ cargo i18n -h -cargo-i18n 0.5.0 +cargo-i18n 3.1.0 --------------------------------------- -Rust I18n command for help you simply to extract all untranslated texts from soruce code. +Rust I18n command to help you extract all untranslated texts from source code. -It will iter all Rust files in and extract all untranslated texts that used `t!` macro. -And then generate a YAML file and merge for existing texts. +It will iterate all Rust files in the source directory and extract all untranslated texts that used `t!` macro. Then it will generate a YAML file and merge with the existing translations. https://github.com/longbridgeapp/rust-i18n -USAGE: - cargo i18n [OPTIONS] [--] [source] +Usage: cargo i18n [OPTIONS] [-- ] + +Arguments: + [SOURCE] + Extract all untranslated I18n texts from source code + + [default: ./] + +Options: + -t, --translate ... + Manually add a translation to the localization file. + + This is useful for non-literal values in the `t!` macro. + + For example, if you have `t!(format!("Hello, {}!", "world"))` in your code, + you can add a translation for it using `-t "Hello, world!"`, + or provide a translated message using `-t "Hello, world! => Hola, world!"`. + + NOTE: The whitespace before and after the key and value will be trimmed. -FLAGS: - -h, --help Prints help information - -V, --version Prints version information + -h, --help + Print help (see a summary with '-h') -ARGS: - Path of your Rust crate root [default: ./] + -V, --version + Print version ``` ## Debugging the Codegen Process @@ -371,7 +431,7 @@ $ RUST_I18N_DEBUG=1 cargo build ## Benchmark -Benchmark `t!` method, result on Apple M1: +Benchmark [`t!`] method, result on Apple M1: ```bash t time: [58.274 ns 60.222 ns 62.390 ns] diff --git a/benches/minify_key.rs b/benches/minify_key.rs new file mode 100644 index 0000000..3e0425f --- /dev/null +++ b/benches/minify_key.rs @@ -0,0 +1,110 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use rust_i18n::t; + +rust_i18n::i18n!("./tests/locales", minify_key = true, minify_key_len = 12); + +pub fn bench_t(c: &mut Criterion) { + c.bench_function("t", |b| b.iter(|| t!("hello"))); + + c.bench_function("t_lorem_ipsum", |b| b.iter(|| t!( + r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. + + Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# + ))); + + c.bench_function("t_with_locale", |b| b.iter(|| t!("hello", locale = "en"))); + + c.bench_function("tr_with_threads", |b| { + let exit_loop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut handles = Vec::new(); + for _ in 0..4 { + let exit_loop = exit_loop.clone(); + handles.push(std::thread::spawn(move || { + while !exit_loop.load(std::sync::atomic::Ordering::SeqCst) { + criterion::black_box(t!("hello")); + } + })); + } + b.iter(|| t!("hello")); + exit_loop.store(true, std::sync::atomic::Ordering::SeqCst); + for handle in handles { + handle.join().unwrap(); + } + }); + + c.bench_function("tr_with_args", |b| { + b.iter(|| { + t!( + "Hello, %{name}. Your message is: %{msg}", + name = "Jason", + msg = "Bla bla" + ) + }) + }); + + c.bench_function("tr_with_args (str)", |b| { + b.iter(|| { + t!( + "Hello, %{name}. Your message is: %{msg}", + "name" = "Jason", + "msg" = "Bla bla" + ) + }) + }); + + c.bench_function("tr_with_args (many)", |b| { + b.iter(|| { + t!( + r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. + You live in %{city} %{zip}. + Your website is %{website}."#, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ) + }) + }); + + c.bench_function("t_with_args (many-dynamic)", |b| { + let msg = r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. + You live in %{city} %{zip}. + Your website is %{website}."# + .to_string(); + b.iter(|| { + t!( + &msg, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ) + }) + }); + + c.bench_function("format! (many)", |b| { + b.iter(|| { + format!( + r#"Hello {name} %{surname}, your account id is {id}, email address is {email}. + You live in {city} {zip}. + Your website is {website}."#, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ); + }) + }); +} + +criterion_group!(benches, bench_t); +criterion_main!(benches); diff --git a/build.rs b/build.rs index 063b9be..8e65a2b 100644 --- a/build.rs +++ b/build.rs @@ -25,7 +25,7 @@ fn main() { let workdir = workdir().unwrap_or("./".to_string()); let locale_path = format!("{workdir}/**/locales/**/*"); - if let Ok(globs) = globwalk::glob(&locale_path) { + if let Ok(globs) = globwalk::glob(locale_path) { for entry in globs { if let Err(e) = entry { println!("cargo:i18n-error={}", e); diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 99a1440..d397d72 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -10,11 +10,8 @@ version = "3.0.0" [dependencies] anyhow = "1" clap = { version = "4.1.14", features = ["derive"] } -itertools = "0.11.0" rust-i18n-support = { path = "../support", version = "3.0.0" } rust-i18n-extract = { path = "../extract", version = "3.0.0" } -serde = { version = "1", features = ["derive"] } -toml = "0.7.4" [[bin]] name = "cargo-i18n" diff --git a/crates/cli/src/config.rs b/crates/cli/src/config.rs deleted file mode 100644 index 883a433..0000000 --- a/crates/cli/src/config.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! This crate defines `struct`s that can be deserialized with Serde -//! to load and inspect `Cargo.toml` metadata. -//! -//! See `Manifest::from_slice`. - -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use std::fs; -use std::io; -use std::io::Read; -use std::path::Path; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct I18nConfig { - #[serde(default = "default_locale")] - pub default_locale: String, - #[serde(default = "available_locales")] - pub available_locales: Vec, - #[serde(default = "load_path")] - pub load_path: String, -} - -fn default_locale() -> String { - I18nConfig::default().default_locale -} - -fn available_locales() -> Vec { - I18nConfig::default().available_locales -} - -fn load_path() -> String { - I18nConfig::default().load_path -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "kebab-case")] -pub struct MainConfig { - pub i18n: I18nConfig, -} - -impl Default for I18nConfig { - fn default() -> Self { - I18nConfig { - default_locale: "en".to_string(), - available_locales: vec!["en".to_string()], - load_path: "./locales".to_string(), - } - } -} - -pub fn load(cargo_root: &Path) -> io::Result { - let cargo_file = cargo_root.join("Cargo.toml"); - let mut file = fs::File::open(&cargo_file) - .unwrap_or_else(|e| panic!("Fail to open {}, {}", cargo_file.display(), e)); - - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - - parse(&contents) -} - -pub fn parse(contents: &str) -> io::Result { - if !contents.contains("[i18n]") && !contents.contains("[package.metadata.i18n]") { - return Ok(I18nConfig::default()); - } - let contents = contents.replace("[package.metadata.i18n]", "[i18n]"); - let mut config: MainConfig = toml::from_str(&contents) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; - - // Push default_locale - config - .i18n - .available_locales - .insert(0, config.i18n.default_locale.clone()); - - // unqiue - config.i18n.available_locales = config.i18n.available_locales.into_iter().unique().collect(); - - Ok(config.i18n) -} - -#[test] -fn test_parse() { - let contents = r#" - [i18n] - default-locale = "en" - available-locales = ["zh-CN"] - load-path = "./my-locales" - "#; - - let cfg = parse(contents).unwrap(); - assert_eq!(cfg.default_locale, "en"); - assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); - assert_eq!(cfg.load_path, "./my-locales"); - - let contents = r#" - [i18n] - available-locales = ["zh-CN", "de", "de"] - load-path = "./my-locales" - "#; - let cfg = parse(contents).unwrap(); - assert_eq!(cfg.default_locale, "en"); - assert_eq!(cfg.available_locales, vec!["en", "zh-CN", "de"]); - assert_eq!(cfg.load_path, "./my-locales"); - - let contents = ""; - let cfg = parse(contents).unwrap(); - assert_eq!(cfg.default_locale, "en"); - assert_eq!(cfg.available_locales, vec!["en"]); - assert_eq!(cfg.load_path, "./locales"); -} - -#[test] -fn test_parse_with_metadata() { - let contents = r#" - [package.metadata.i18n] - default-locale = "en" - available-locales = ["zh-CN"] - load-path = "./my-locales" - "#; - - let cfg = parse(contents).unwrap(); - assert_eq!(cfg.default_locale, "en"); - assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); - assert_eq!(cfg.load_path, "./my-locales"); -} - -#[test] -fn test_load_default() { - let workdir = Path::new(env!["CARGO_MANIFEST_DIR"]); - - let cfg = load(workdir).unwrap(); - assert_eq!(cfg.default_locale, "en"); - assert_eq!(cfg.available_locales, vec!["en"]); - assert_eq!(cfg.load_path, "./locales"); -} - -#[test] -fn test_load() { - let workdir = Path::new(env!["CARGO_MANIFEST_DIR"]); - let cargo_root = workdir.join("../../examples/foo"); - - let cfg = load(&cargo_root).unwrap(); - assert_eq!(cfg.default_locale, "en"); - assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); -} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 6690627..d33932e 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,10 +1,9 @@ use anyhow::Error; use clap::{Args, Parser}; - -use std::{collections::HashMap, path::Path}; - +use rust_i18n_extract::extractor::Message; use rust_i18n_extract::{extractor, generator, iter}; -mod config; +use rust_i18n_support::{I18nConfig, MinifyKey}; +use std::{collections::HashMap, path::Path}; #[derive(Parser)] #[command(name = "cargo")] @@ -24,11 +23,79 @@ enum CargoCli { /// /// https://github.com/longbridgeapp/rust-i18n struct I18nArgs { + /// Manually add a translation to the localization file. + /// + /// This is useful for non-literal values in the `t!` macro. + /// + /// For example, if you have `t!(format!("Hello, {}!", "world"))` in your code, + /// you can add a translation for it using `-t "Hello, world!"`, + /// or provide a translated message using `-t "Hello, world! => Hola, world!"`. + /// + /// NOTE: The whitespace before and after the key and value will be trimmed. + #[arg(short, long, default_value = None, name = "TEXT", num_args(1..), value_parser = translate_value_parser, verbatim_doc_comment)] + translate: Option>, /// Extract all untranslated I18n texts from source code - #[arg(default_value = "./")] + #[arg(default_value = "./", last = true)] source: Option, } +/// Remove quotes from a string at the start and end. +fn remove_quotes(s: &str) -> &str { + let mut start = 0; + let mut end = s.len(); + if s.starts_with('"') { + start += 1; + } + if s.ends_with('"') { + end -= 1; + } + &s[start..end] +} + +/// Parse a string of the form "key => value" into a tuple. +fn translate_value_parser(s: &str) -> Result<(String, String), std::io::Error> { + if let Some((key, msg)) = s.split_once("=>") { + let key = remove_quotes(key.trim()); + let msg = remove_quotes(msg.trim()); + Ok((key.to_owned(), msg.to_owned())) + } else { + Ok((s.to_owned(), s.to_owned())) + } +} + +/// Add translations to the localize file for tr! +fn add_translations( + list: &[(String, String)], + results: &mut HashMap, + cfg: &I18nConfig, +) { + let I18nConfig { + minify_key, + minify_key_len, + minify_key_prefix, + minify_key_thresh, + .. + } = cfg; + + for item in list { + let index = results.len(); + let key = if *minify_key { + let hashed_key = + item.0 + .minify_key(*minify_key_len, minify_key_prefix, *minify_key_thresh); + hashed_key.to_string() + } else { + item.0.clone() + }; + results.entry(key).or_insert(Message { + key: item.1.clone(), + index, + minify_key: *minify_key, + locations: vec![], + }); + } +} + fn main() -> Result<(), Error> { let CargoCli::I18n(args) = CargoCli::parse(); @@ -36,14 +103,18 @@ fn main() -> Result<(), Error> { let source_path = args.source.expect("Missing source path"); - let cfg = config::load(std::path::Path::new(&source_path))?; + let cfg = I18nConfig::load(std::path::Path::new(&source_path))?; iter::iter_crate(&source_path, |path, source| { - extractor::extract(&mut results, path, source) + extractor::extract(&mut results, path, source, cfg.clone()) })?; - let mut messages: Vec<_> = results.values().collect(); - messages.sort_by_key(|m| m.index); + if let Some(list) = args.translate { + add_translations(&list, &mut results, &cfg); + } + + let mut messages: Vec<_> = results.iter().collect(); + messages.sort_by_key(|(_k, m)| m.index); let mut has_error = false; diff --git a/crates/extract/src/extractor.rs b/crates/extract/src/extractor.rs index 613bced..a64be79 100644 --- a/crates/extract/src/extractor.rs +++ b/crates/extract/src/extractor.rs @@ -1,6 +1,7 @@ use anyhow::Error; use proc_macro2::{TokenStream, TokenTree}; use quote::ToTokens; +use rust_i18n_support::I18nConfig; use std::collections::HashMap; use std::path::PathBuf; @@ -16,24 +17,31 @@ pub struct Location { pub struct Message { pub key: String, pub index: usize, + pub minify_key: bool, pub locations: Vec, } impl Message { - fn new(key: &str, index: usize) -> Self { + fn new(key: &str, index: usize, minify_key: bool) -> Self { Self { key: key.to_owned(), index, + minify_key, locations: vec![], } } } -static METHOD_NAME: &str = "t"; +static METHOD_NAMES: &[&str] = &["t", "tr"]; #[allow(clippy::ptr_arg)] -pub fn extract(results: &mut Results, path: &PathBuf, source: &str) -> Result<(), Error> { - let mut ex = Extractor { results, path }; +pub fn extract( + results: &mut Results, + path: &PathBuf, + source: &str, + cfg: I18nConfig, +) -> Result<(), Error> { + let mut ex = Extractor { results, path, cfg }; let file = syn::parse_file(source) .unwrap_or_else(|_| panic!("Failed to parse file, file: {}", path.display())); @@ -45,6 +53,7 @@ pub fn extract(results: &mut Results, path: &PathBuf, source: &str) -> Result<() struct Extractor<'a> { results: &'a mut Results, path: &'a PathBuf, + cfg: I18nConfig, } impl<'a> Extractor<'a> { @@ -63,7 +72,8 @@ impl<'a> Extractor<'a> { } } - if ident == METHOD_NAME && is_macro { + let ident_str = ident.to_string(); + if METHOD_NAMES.contains(&ident_str.as_str()) && is_macro { if let Some(TokenTree::Group(group)) = token_iter.peek() { self.take_message(group.stream()); } @@ -85,17 +95,34 @@ impl<'a> Extractor<'a> { return; }; + let I18nConfig { + minify_key, + minify_key_len, + minify_key_prefix, + minify_key_thresh, + .. + } = &self.cfg; let key: Option = Some(literal); if let Some(lit) = key { if let Some(key) = literal_to_string(&lit) { - let message_key = format_message_key(&key); - + let (message_key, message_content) = if *minify_key { + let hashed_key = rust_i18n_support::MinifyKey::minify_key( + &key, + *minify_key_len, + minify_key_prefix, + *minify_key_thresh, + ); + (hashed_key.to_string(), key.clone()) + } else { + let message_key = format_message_key(&key); + (message_key.clone(), message_key) + }; let index = self.results.len(); let message = self .results - .entry(message_key.clone()) - .or_insert_with(|| Message::new(&message_key, index)); + .entry(message_key) + .or_insert_with(|| Message::new(&message_content, index, *minify_key)); let span = lit.span(); let line = span.start().line; @@ -143,6 +170,7 @@ mod tests { )+ ], index: 0, + minify_key: false, }; results.push(message); )+ @@ -211,6 +239,7 @@ mod tests { let mut ex = Extractor { results: &mut results, path: &"hello.rs".to_owned().into(), + cfg: I18nConfig::default(), }; ex.invoke(stream).unwrap(); diff --git a/crates/extract/src/generator.rs b/crates/extract/src/generator.rs index 4fc2ffc..db81e4f 100644 --- a/crates/extract/src/generator.rs +++ b/crates/extract/src/generator.rs @@ -10,7 +10,7 @@ type Translations = HashMap>; pub fn generate<'a, P: AsRef>( output_path: P, all_locales: &Vec, - messages: impl IntoIterator + Clone, + messages: impl IntoIterator + Clone, ) -> Result<()> { let filename = "TODO.yml"; let format = "yaml"; @@ -63,7 +63,7 @@ fn generate_result<'a, P: AsRef>( output_path: P, output_filename: &str, all_locales: &Vec, - messages: impl IntoIterator + Clone, + messages: impl IntoIterator + Clone, ) -> Translations { let mut trs = Translations::new(); @@ -76,7 +76,7 @@ fn generate_result<'a, P: AsRef>( let ignore_file = |fname: &str| fname.ends_with(&output_filename); let data = load_locales(&output_path, ignore_file); - for m in messages.clone() { + for (key, m) in messages.clone() { if !m.locations.is_empty() { for _l in &m.locations { // TODO: write file and line as YAML comment @@ -86,14 +86,18 @@ fn generate_result<'a, P: AsRef>( } if let Some(trs) = data.get(locale) { - if trs.get(&m.key).is_some() { + if trs.get(key).is_some() { continue; } } - let value = m.key.split('.').last().unwrap_or_default(); + let value = if m.minify_key { + m.key.to_owned() + } else { + m.key.split('.').last().unwrap_or_default().to_string() + }; - trs.entry(m.key.clone()) + trs.entry(key.clone()) .or_insert_with(HashMap::new) .insert(locale.to_string(), value.to_string()); } diff --git a/crates/macro/Cargo.toml b/crates/macro/Cargo.toml index 0050a4c..2661b3e 100644 --- a/crates/macro/Cargo.toml +++ b/crates/macro/Cargo.toml @@ -18,10 +18,13 @@ rust-i18n-support = { path = "../support", version = "3.0.0" } serde = "1" serde_json = "1" serde_yaml = "0.8" -syn = { version = "2.0.18", features = ["full"] } +syn = { version = "2.0.18", features = ["full", "extra-traits"] } [dev-dependencies] rust-i18n = { path = "../.." } [lib] proc-macro = true + +[features] +log-miss-tr = [] diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index 30d23ff..78beb85 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -1,12 +1,24 @@ use quote::quote; -use rust_i18n_support::{is_debug, load_locales}; +use rust_i18n_support::{ + is_debug, load_locales, I18nConfig, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, + DEFAULT_MINIFY_KEY_PREFIX, DEFAULT_MINIFY_KEY_THRESH, +}; use std::collections::HashMap; -use syn::{parse_macro_input, Expr, Ident, LitStr, Token}; +use syn::{parse_macro_input, Expr, Ident, LitBool, LitStr, Token}; + +mod mikey; +mod tr; struct Args { locales_path: String, + default_locale: Option, fallback: Option>, extend: Option, + metadata: bool, + minify_key: bool, + minify_key_len: usize, + minify_key_prefix: String, + minify_key_thresh: usize, } impl Args { @@ -44,6 +56,60 @@ impl Args { Ok(()) } + fn consume_metadata(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { + let lit_bool = input.parse::()?; + self.metadata = lit_bool.value; + // Load the config from Cargo.toml. This can be overridden by subsequent options. + if self.metadata { + // CARGO_MANIFEST_DIR is current build directory + let cargo_dir = std::env::var("CARGO_MANIFEST_DIR") + .map_err(|_| input.error("The CARGO_MANIFEST_DIR is required fo `metadata`"))?; + let current_dir = std::path::PathBuf::from(cargo_dir); + let cfg = I18nConfig::load(¤t_dir) + .map_err(|_| input.error("Failed to load config from Cargo.toml for `metadata`"))?; + self.locales_path = cfg.load_path; + self.default_locale = Some(cfg.default_locale.clone()); + if !cfg.fallback.is_empty() { + self.fallback = Some(cfg.fallback); + } + self.minify_key = cfg.minify_key; + self.minify_key_len = cfg.minify_key_len; + self.minify_key_prefix = cfg.minify_key_prefix; + self.minify_key_thresh = cfg.minify_key_thresh; + } + Ok(()) + } + + fn consume_minify_key(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { + let lit_bool = input.parse::()?; + self.minify_key = lit_bool.value; + Ok(()) + } + + fn consume_minify_key_len(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { + let lit_int = input.parse::()?; + self.minify_key_len = lit_int.base10_parse()?; + Ok(()) + } + + fn consume_minify_key_prefix( + &mut self, + input: syn::parse::ParseStream, + ) -> syn::parse::Result<()> { + let lit_str = input.parse::()?; + self.minify_key_prefix = lit_str.value(); + Ok(()) + } + + fn consume_minify_key_thresh( + &mut self, + input: syn::parse::ParseStream, + ) -> syn::parse::Result<()> { + let lit_int = input.parse::()?; + self.minify_key_thresh = lit_int.base10_parse()?; + Ok(()) + } + fn consume_options(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { let ident = input.parse::()?.to_string(); input.parse::()?; @@ -56,6 +122,21 @@ impl Args { let val = input.parse::()?; self.extend = Some(val); } + "metadata" => { + self.consume_metadata(input)?; + } + "minify_key" => { + self.consume_minify_key(input)?; + } + "minify_key_len" => { + self.consume_minify_key_len(input)?; + } + "minify_key_prefix" => { + self.consume_minify_key_prefix(input)?; + } + "minify_key_thresh" => { + self.consume_minify_key_thresh(input)?; + } _ => {} } @@ -85,6 +166,16 @@ impl syn::parse::Parse for Args { /// # fn v4() { /// i18n!("locales", fallback = ["en", "es"]); /// # } + /// # fn v5() { + /// i18n!("locales", fallback = ["en", "es"], + /// minify_key = true, + /// minify_key_len = 12, + /// minify_key_prefix = "T.", + /// minify_key_thresh = 64); + /// # } + /// # fn v6() { + /// i18n!(metadata = true); + /// # } /// ``` /// /// Ref: https://docs.rs/syn/latest/syn/parse/index.html @@ -93,8 +184,14 @@ impl syn::parse::Parse for Args { let mut result = Self { locales_path: String::from("locales"), + default_locale: None, fallback: None, extend: None, + metadata: false, + minify_key: DEFAULT_MINIFY_KEY, + minify_key_len: DEFAULT_MINIFY_KEY_LEN, + minify_key_prefix: DEFAULT_MINIFY_KEY_PREFIX.to_owned(), + minify_key_thresh: DEFAULT_MINIFY_KEY_THRESH, }; if lookahead.peek(LitStr) { @@ -115,7 +212,17 @@ impl syn::parse::Parse for Args { /// /// This will load all translations by glob `**/*.yml` from the given path, default: `${CARGO_MANIFEST_DIR}/locales`. /// -/// Attribute `fallback` for set the fallback locale, if present `t` macro will use it as the fallback locale. +/// # Attributes +/// +/// - `fallback` for set the fallback locale, if present [`t!`](macro.t.html) macro will use it as the fallback locale. +/// - `backend` for set the backend, if present [`t!`](macro.t.html) macro will use it as the backend. +/// - `metadata` to enable/disable loading of the [package.metadata.i18n] config from Cargo.toml, default: `false`. +/// - `minify_key` for enable/disable minify key, default: [`DEFAULT_MINIFY_KEY`](constant.DEFAULT_MINIFY_KEY.html). +/// - `minify_key_len` for set the minify key length, default: [`DEFAULT_MINIFY_KEY_LEN`](constant.DEFAULT_MINIFY_KEY_LEN.html), +/// * The range of available values is from `0` to `24`. +/// - `minify_key_prefix` for set the minify key prefix, default: [`DEFAULT_MINIFY_KEY_PREFIX`](constant.DEFAULT_MINIFY_KEY_PREFIX.html). +/// - `minify_key_thresh` for set the minify key threshold, default: [`DEFAULT_MINIFY_KEY_THRESH`](constant.DEFAULT_MINIFY_KEY_THRESH.html). +/// * If the length of the value is less than or equal to this value, the value will not be minified. /// /// ```no_run /// # use rust_i18n::i18n; @@ -131,6 +238,16 @@ impl syn::parse::Parse for Args { /// # fn v4() { /// i18n!("locales", fallback = ["en", "es"]); /// # } +/// # fn v5() { +/// i18n!("locales", fallback = ["en", "es"], +/// minify_key = true, +/// minify_key_len = 12, +/// minify_key_prefix = "T.", +/// minify_key_thresh = 64); +/// # } +/// # fn v6() { +/// i18n!(metadata = true); +/// # } /// ``` #[proc_macro] pub fn i18n(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -177,6 +294,14 @@ fn generate_code( }); }); + let default_locale = if let Some(default_locale) = args.default_locale { + quote! { + rust_i18n::set_locale(#default_locale); + } + } else { + quote! {} + }; + let fallback = if let Some(fallback) = args.fallback { quote! { Some(&[#(#fallback),*]) @@ -195,9 +320,13 @@ fn generate_code( quote! {} }; - // result + let minify_key = args.minify_key; + let minify_key_len = args.minify_key_len; + let minify_key_prefix = args.minify_key_prefix; + let minify_key_thresh = args.minify_key_thresh; + quote! { - use rust_i18n::BackendExt; + use rust_i18n::{BackendExt, CowStr, MinifyKey}; use std::borrow::Cow; /// I18n backend instance @@ -209,10 +338,16 @@ fn generate_code( #(#all_translations)* #extend_code + #default_locale + Box::new(backend) }); static _RUST_I18N_FALLBACK_LOCALE: Option<&[&'static str]> = #fallback; + static _RUST_I18N_MINIFY_KEY: bool = #minify_key; + static _RUST_I18N_MINIFY_KEY_LEN: usize = #minify_key_len; + static _RUST_I18N_MINIFY_KEY_PREFIX: &str = #minify_key_prefix; + static _RUST_I18N_MINIFY_KEY_THRESH: usize = #minify_key_thresh; /// Lookup fallback locales /// @@ -229,30 +364,34 @@ fn generate_code( #[inline] #[allow(missing_docs)] pub fn _rust_i18n_translate<'r>(locale: &str, key: &'r str) -> Cow<'r, str> { - if let Some(value) = _RUST_I18N_BACKEND.translate(locale, key) { - return value.into(); - } - - let mut current_locale = locale; - while let Some(fallback_locale) = _rust_i18n_lookup_fallback(current_locale) { - if let Some(value) = _RUST_I18N_BACKEND.translate(fallback_locale, key) { - return value.into(); + _rust_i18n_try_translate(locale, key).unwrap_or_else(|| { + if locale.is_empty() { + key.into() + } else { + format!("{}.{}", locale, key).into() } - current_locale = fallback_locale; - } + }) + } - if let Some(fallback) = _RUST_I18N_FALLBACK_LOCALE { - for locale in fallback { - if let Some(value) = _RUST_I18N_BACKEND.translate(locale, key) { - return value.into(); + /// Try to get I18n text by locale and key + #[inline] + #[allow(missing_docs)] + pub fn _rust_i18n_try_translate<'r>(locale: &str, key: impl AsRef) -> Option> { + _RUST_I18N_BACKEND.translate(locale, key.as_ref()) + .map(Cow::from) + .or_else(|| { + let mut current_locale = locale; + while let Some(fallback_locale) = _rust_i18n_lookup_fallback(current_locale) { + if let Some(value) = _RUST_I18N_BACKEND.translate(fallback_locale, key.as_ref()) { + return Some(Cow::from(value)); + } + current_locale = fallback_locale; } - } - } - if locale.is_empty() { - return key.into(); - } - return format!("{}.{}", locale, key).into(); + _RUST_I18N_FALLBACK_LOCALE.and_then(|fallback| { + fallback.iter().find_map(|locale| _RUST_I18N_BACKEND.translate(locale, key.as_ref()).map(Cow::from)) + }) + }) } #[allow(missing_docs)] @@ -261,11 +400,58 @@ fn generate_code( locales.sort(); locales } + + #[allow(unused_macros)] + macro_rules! __rust_i18n_t { + ($($all_tokens:tt)*) => { + rust_i18n::tr!($($all_tokens)*, _minify_key = #minify_key, _minify_key_len = #minify_key_len, _minify_key_prefix = #minify_key_prefix, _minify_key_thresh = #minify_key_thresh) + } + } + + #[allow(unused_macros)] + macro_rules! __rust_i18n_tkv { + ($msg:literal) => { + { + let val = $msg; + let key = rust_i18n::mikey!($msg, #minify_key_len, #minify_key_prefix, #minify_key_thresh); + (key, val) + } + } + } + + pub(crate) use __rust_i18n_t as _rust_i18n_t; + pub(crate) use __rust_i18n_tkv as _rust_i18n_tkv; } } +/// A procedural macro that generates a string representation of the input. +/// +/// This macro accepts either a string literal or an identifier as input. +/// If the input is a string literal, it returns the value of the string literal. +/// If the input is an identifier, it returns the string representation of the identifier. +/// +/// # Arguments +/// +/// * `input` - The input token stream. It should be either a string literal or an identifier. +/// +/// # Returns +/// +/// Returns a token stream that contains a string representation of the input. If the input cannot be parsed as a string literal or an identifier, +/// it returns a compile error. +/// +/// # Example +/// +/// ```no_run +/// # use rust_i18n::vakey; +/// # fn v1() { +/// let key = vakey!(name); +/// # } +/// # fn v2() { +/// let key = vakey!("name"); +/// # } +/// ``` #[proc_macro] -pub fn key(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +pub fn vakey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let output = syn::parse::(input.clone()) .map(|str| str.value()) .or(syn::parse::(input.clone()).map(|ident| format!("{}", ident))); @@ -275,3 +461,76 @@ pub fn key(input: proc_macro::TokenStream) -> proc_macro::TokenStream { Err(err) => err.to_compile_error().into(), } } + +/// A procedural macro that generates a translation key from a value. +/// +/// # Arguments +/// +/// * `value` - The value to be generated. +/// * `key_len` - The length of the translation key. +/// * `prefix` - The prefix of the translation key. +/// * `threshold` - The minimum length of the value to be generated. +/// +/// # Returns +/// +/// * If `value.len() <= threshold` then returns the origin value. +/// * Otherwise, returns a base62 encoded 128 bits hashed translation key. +/// +/// # Example +/// +/// ```no_run +/// # use rust_i18n::mikey; +/// # fn v1() { +/// mikey!("Hello world", 12, "T.", 64); +/// // => "Hello world" +/// +/// mikey!("Hello world", 12, "T.", 5); +/// // => "T.1b9d6bcd" +/// # } +/// ``` +#[proc_macro] +pub fn mikey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + parse_macro_input!(input as mikey::MiKey).into() +} + +/// A procedural macro that retrieves the i18n text for the `t!` macro. +/// +/// This macro first checks if a translation exists for the input string. +/// If it does, it returns the translated string. +/// If it does not, it returns the input value. +/// +/// # Examples +/// +/// ```no_run +/// #[macro_use] extern crate rust_i18n; +/// # use rust_i18n::{tr, CowStr}; +/// # fn _rust_i18n_try_translate<'r>(locale: &str, key: &'r str) -> Option> { todo!() } +/// # fn main() { +/// // Simple get text with current locale +/// tr!("Hello world"); +/// // => "Hello world" +/// +/// // Get a special locale's text +/// tr!("Hello world", locale = "de"); +/// // => "Hallo Welt!" +/// +/// // With variables +/// tr!("Hello, %{name}", name = "world"); // Asignment style +/// tr!("Hello, %{name}", name => "world"); // Arrow style +/// // => "Hello, world" +/// tr!("Hello, %{name} and %{other}", name = "Foo", other = "Bar"); +/// // => "Hello, Foo and Bar" +/// +/// // With variables and specifiers +/// tr!("Hello, %{name} and %{other}", name = "Foo", other = 123 : {:08}); +/// // => "Hello, Foo and 00000123" +/// +/// // With locale and variables +/// tr!("Hallo, %{name}", locale = "de", name => "Jason"); +/// // => "Hallo, Jason" +/// # } +/// ``` +#[proc_macro] +pub fn tr(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + parse_macro_input!(input as tr::Tr).into() +} diff --git a/crates/macro/src/mikey.rs b/crates/macro/src/mikey.rs new file mode 100644 index 0000000..e71939c --- /dev/null +++ b/crates/macro/src/mikey.rs @@ -0,0 +1,49 @@ +use quote::quote; +use rust_i18n_support::minify_key; +use syn::Token; + +/// A type representing the `mikey!` proc macro. +#[derive(Clone, Debug, Default)] +pub struct MiKey { + msg: String, + len: usize, + prefix: String, + threshold: usize, +} + +impl MiKey { + fn into_token_stream(self) -> proc_macro2::TokenStream { + let key = minify_key(&self.msg, self.len, &self.prefix, self.threshold); + quote! { #key } + } +} + +impl syn::parse::Parse for MiKey { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + let msg = input.parse::()?.value(); + let _comma = input.parse::()?; + let len: usize = input.parse::()?.base10_parse()?; + let _comma = input.parse::()?; + let prefix = input.parse::()?.value(); + let _comma = input.parse::()?; + let threshold: usize = input.parse::()?.base10_parse()?; + Ok(Self { + msg, + len, + prefix, + threshold, + }) + } +} + +impl From for proc_macro::TokenStream { + fn from(val: MiKey) -> Self { + val.into_token_stream().into() + } +} + +impl From for proc_macro2::TokenStream { + fn from(val: MiKey) -> Self { + val.into_token_stream() + } +} diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs new file mode 100644 index 0000000..77b479a --- /dev/null +++ b/crates/macro/src/tr.rs @@ -0,0 +1,506 @@ +use quote::{quote, ToTokens}; +use rust_i18n_support::{ + MinifyKey, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, DEFAULT_MINIFY_KEY_THRESH, +}; +use syn::{parse::discouraged::Speculative, token::Brace, Expr, Ident, LitStr, Token}; + +#[derive(Clone, Debug, Default)] +pub enum Value { + #[default] + Empty, + Expr(Expr), + Ident(Ident), +} + +impl Value { + fn is_expr_lit_str(&self) -> bool { + if let Self::Expr(Expr::Lit(expr_lit)) = self { + if let syn::Lit::Str(_) = &expr_lit.lit { + return true; + } + } + false + } + + fn is_expr_tuple(&self) -> bool { + if let Self::Expr(Expr::Tuple(_)) = self { + return true; + } + false + } + + fn to_string(&self) -> Option { + if let Self::Expr(Expr::Lit(expr_lit)) = self { + if let syn::Lit::Str(lit_str) = &expr_lit.lit { + return Some(lit_str.value()); + } + } + None + } + + fn to_tupled_token_streams( + &self, + ) -> syn::parse::Result<(proc_macro2::TokenStream, proc_macro2::TokenStream)> { + if let Self::Expr(Expr::Tuple(expr_tuple)) = self { + if expr_tuple.elems.len() == 2 { + let first = expr_tuple.elems.first().map(|v| quote! { #v }).unwrap(); + let last = expr_tuple.elems.last().map(|v| quote! { #v }).unwrap(); + return Ok((first, last)); + } + } + Err(syn::Error::new_spanned( + self, + "Expected a tuple with two elements", + )) + } +} + +impl From for Value { + fn from(expr: Expr) -> Self { + Self::Expr(expr) + } +} + +impl From for Value { + fn from(ident: Ident) -> Self { + Self::Ident(ident) + } +} + +impl quote::ToTokens for Value { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + match self { + Self::Empty => {} + Self::Expr(expr) => match expr { + Expr::Path(path) => quote! { &#path }.to_tokens(tokens), + expr => expr.to_tokens(tokens), + }, + Self::Ident(ident) => quote! { &#ident }.to_tokens(tokens), + } + } +} + +impl syn::parse::Parse for Value { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + if let Ok(expr) = fork.parse::() { + input.advance_to(&fork); + return Ok(expr.into()); + } + let fork = input.fork(); + if let Ok(expr) = fork.parse::() { + input.advance_to(&fork); + return Ok(expr.into()); + } + Err(input.error("Expected a expression or an identifier")) + } +} + +#[derive(Clone, Default)] +pub struct Argument { + pub name: String, + pub value: Value, + pub specifiers: Option, +} + +impl Argument { + #[allow(dead_code)] + pub fn value_string(&self) -> String { + match &self.value { + Value::Expr(Expr::Lit(expr_lit)) => match &expr_lit.lit { + syn::Lit::Str(lit_str) => lit_str.value(), + _ => self.value.to_token_stream().to_string(), + }, + _ => self.value.to_token_stream().to_string(), + } + } + + fn try_ident(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let ident = fork.parse::()?; + input.advance_to(&fork); + Ok(ident.to_string()) + } + + fn try_literal(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let lit = fork.parse::()?; + input.advance_to(&fork); + Ok(lit.value()) + } +} + +impl syn::parse::Parse for Argument { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + // Ignore leading commas. + while input.peek(Token![,]) { + let _ = input.parse::()?; + } + // Parse the argument name. + let name = Self::try_ident(input) + .or_else(|_| Self::try_literal(input)) + .map_err(|_| input.error("Expected a `string` literal or an identifier"))?; + // Parse the separator between the name and the value. + if input.peek(Token![=>]) { + let _ = input.parse::]>()?; + } else if input.peek(Token![=]) { + let _ = input.parse::()?; + } else { + return Err(input.error("Expected `=>` or `=`")); + } + // Parse the argument value. + let value = input.parse()?; + // Parse the specifiers [optinal]. + let specifiers = if input.peek(Token![:]) { + let _ = input.parse::()?; + if input.peek(Brace) { + let content; + let _ = syn::braced!(content in input); + let mut specifiers = String::new(); + while let Ok(s) = content.parse::() { + specifiers.push_str(&s.to_string()); + } + Some(specifiers) + } else { + None + } + } else { + None + }; + Ok(Self { + name, + value, + specifiers, + }) + } +} + +#[derive(Default)] +pub struct Arguments { + pub args: Vec, +} + +impl Arguments { + pub fn is_empty(&self) -> bool { + self.args.is_empty() + } + + pub fn iter(&self) -> impl Iterator { + self.args.iter() + } + + pub fn keys(&self) -> Vec { + self.args.iter().map(|arg| arg.name.clone()).collect() + } + + #[allow(dead_code)] + pub fn values(&self) -> Vec { + self.args.iter().map(|arg| arg.value.clone()).collect() + } +} + +impl AsRef> for Arguments { + fn as_ref(&self) -> &Vec { + &self.args + } +} + +impl AsMut> for Arguments { + fn as_mut(&mut self) -> &mut Vec { + &mut self.args + } +} + +impl syn::parse::Parse for Arguments { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let args = input + .parse_terminated(Argument::parse, Token![,])? + .into_iter() + .collect(); + Ok(Self { args }) + } +} + +#[derive(Default)] +pub struct Messsage { + pub key: proc_macro2::TokenStream, + pub val: Value, +} + +impl Messsage { + fn try_exp(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let expr = fork.parse::()?; + input.advance_to(&fork); + + Ok(Self { + key: Default::default(), + val: Value::Expr(expr), + }) + } + + fn try_ident(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let ident = fork.parse::()?; + input.advance_to(&fork); + Ok(Self { + key: Default::default(), + val: Value::Ident(ident), + }) + } +} + +impl syn::parse::Parse for Messsage { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let result = Self::try_exp(input).or_else(|_| Self::try_ident(input))?; + Ok(result) + } +} + +/// A type representing the `tr!` proc macro. +pub(crate) struct Tr { + pub msg: Messsage, + pub args: Arguments, + pub locale: Option, + pub minify_key: bool, + pub minify_key_len: usize, + pub minify_key_prefix: String, + pub minify_key_thresh: usize, +} + +impl Tr { + fn new() -> Self { + Self { + msg: Messsage::default(), + args: Arguments::default(), + locale: None, + minify_key: false, + minify_key_len: DEFAULT_MINIFY_KEY_LEN, + minify_key_prefix: DEFAULT_MINIFY_KEY_PREFIX.into(), + minify_key_thresh: DEFAULT_MINIFY_KEY_THRESH, + } + } + + fn parse_minify_key(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + match &expr_lit.lit { + syn::Lit::Bool(lit_bool) => { + return Ok(lit_bool.value); + } + syn::Lit::Str(lit_str) => { + let value = lit_str.value(); + if ["true", "false", "yes", "no"].contains(&value.as_str()) { + return Ok(["true", "yes"].contains(&value.as_str())); + } + } + _ => {} + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key` Expected a string literal in `true`, `false`, `yes`, `no`", + )) + } + + fn parse_minify_key_len(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + if let syn::Lit::Int(lit_int) = &expr_lit.lit { + return Ok(lit_int.base10_parse().unwrap()); + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key_len` Expected a integer literal", + )) + } + + fn parse_minify_key_prefix(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + if let syn::Lit::Str(lit_str) = &expr_lit.lit { + return Ok(lit_str.value()); + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key_prefix` Expected a string literal", + )) + } + + fn parse_minify_key_thresh(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + if let syn::Lit::Int(lit_int) = &expr_lit.lit { + return Ok(lit_int.base10_parse().unwrap()); + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key_threshold` Expected a integer literal", + )) + } + + fn filter_arguments(&mut self) -> syn::parse::Result<()> { + for arg in self.args.iter() { + match arg.name.as_str() { + "locale" => { + self.locale = Some(arg.value.clone()); + } + "_minify_key" => { + self.minify_key = Self::parse_minify_key(&arg.value)?; + } + "_minify_key_len" => { + self.minify_key_len = Self::parse_minify_key_len(&arg.value)?; + } + "_minify_key_prefix" => { + self.minify_key_prefix = Self::parse_minify_key_prefix(&arg.value)?; + } + "_minify_key_thresh" => { + self.minify_key_thresh = Self::parse_minify_key_thresh(&arg.value)?; + } + _ => {} + } + } + + self.args.as_mut().retain(|v| { + ![ + "locale", + "_minify_key", + "_minify_key_len", + "_minify_key_prefix", + "_minify_key_thresh", + ] + .contains(&v.name.as_str()) + }); + + Ok(()) + } + + #[cfg(feature = "log-miss-tr")] + fn log_missing() -> proc_macro2::TokenStream { + quote! { + log::log!(target: "rust-i18n", log::Level::Warn, "missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); + } + } + + #[cfg(not(feature = "log-miss-tr"))] + fn log_missing() -> proc_macro2::TokenStream { + quote! {} + } + + fn into_token_stream(self) -> proc_macro2::TokenStream { + let (msg_key, msg_val) = if self.minify_key && self.msg.val.is_expr_lit_str() { + let msg_val = self.msg.val.to_string().unwrap(); + let msg_key = MinifyKey::minify_key( + &msg_val, + self.minify_key_len, + self.minify_key_prefix.as_str(), + self.minify_key_thresh, + ); + (quote! { #msg_key }, quote! { #msg_val }) + } else if self.minify_key && self.msg.val.is_expr_tuple() { + self.msg.val.to_tupled_token_streams().unwrap() + } else if self.minify_key { + let minify_key_len = self.minify_key_len; + let minify_key_prefix = self.minify_key_prefix; + let minify_key_thresh = self.minify_key_thresh; + let msg_val = self.msg.val.to_token_stream(); + let msg_key = quote! { rust_i18n::MinifyKey::minify_key(&msg_val, #minify_key_len, #minify_key_prefix, #minify_key_thresh) }; + (msg_key, msg_val) + } else { + let msg_val = self.msg.val.to_token_stream(); + let msg_key = quote! { &msg_val }; + (msg_key, msg_val) + }; + let locale = self.locale.map_or_else( + || quote! { &rust_i18n::locale() }, + |locale| quote! { #locale }, + ); + let keys: Vec<_> = self.args.keys().iter().map(|v| quote! { #v }).collect(); + let values: Vec<_> = self + .args + .as_ref() + .iter() + .map(|v| { + let value = &v.value; + let sepecifiers = v + .specifiers + .as_ref() + .map_or("{}".to_owned(), |s| format!("{{{}}}", s)); + quote! { format!(#sepecifiers, #value) } + }) + .collect(); + let logging = Self::log_missing(); + if self.args.is_empty() { + quote! { + { + let msg_val = #msg_val; + let msg_key = #msg_key; + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { + translated.into() + } else { + #logging + rust_i18n::CowStr::from(msg_val).into_inner() + } + } + } + } else { + quote! { + { + let msg_val = #msg_val; + let msg_key = #msg_key; + let keys = &[#(#keys),*]; + let values = &[#(#values),*]; + { + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { + let replaced = rust_i18n::replace_patterns(&translated, keys, values); + std::borrow::Cow::from(replaced) + } else { + #logging + let replaced = rust_i18n::replace_patterns(rust_i18n::CowStr::from(msg_val).as_str(), keys, values); + std::borrow::Cow::from(replaced) + } + } + } + } + } + } +} + +impl Default for Tr { + fn default() -> Self { + Self::new() + } +} + +impl syn::parse::Parse for Tr { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let msg = input.parse::()?; + let comma = input.parse::>()?; + let args = if comma.is_some() { + input.parse::()? + } else { + Arguments::default() + }; + + let mut result = Self { + msg, + args, + ..Self::new() + }; + + result.filter_arguments()?; + + Ok(result) + } +} + +impl From for proc_macro::TokenStream { + fn from(args: Tr) -> Self { + args.into_token_stream().into() + } +} + +impl From for proc_macro2::TokenStream { + fn from(args: Tr) -> Self { + args.into_token_stream() + } +} diff --git a/crates/support/Cargo.toml b/crates/support/Cargo.toml index ff82f15..a556254 100644 --- a/crates/support/Cargo.toml +++ b/crates/support/Cargo.toml @@ -9,12 +9,15 @@ version = "3.0.1" [dependencies] arc-swap = "1.6.0" +base62 = "2.0.2" globwalk = "0.8.1" +itertools = "0.11.0" once_cell = "1.10.0" proc-macro2 = "1.0" -serde = "1" +serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.8" +siphasher = "1.0" toml = "0.7.4" normpath = "1.1.1" lazy_static = "1" diff --git a/crates/support/src/backend.rs b/crates/support/src/backend.rs index cbea62c..c3ead73 100644 --- a/crates/support/src/backend.rs +++ b/crates/support/src/backend.rs @@ -105,6 +105,12 @@ impl Backend for SimpleBackend { impl BackendExt for SimpleBackend {} +impl Default for SimpleBackend { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/crates/support/src/config.rs b/crates/support/src/config.rs new file mode 100644 index 0000000..7917b4e --- /dev/null +++ b/crates/support/src/config.rs @@ -0,0 +1,209 @@ +//! This crate defines `struct`s that can be deserialized with Serde +//! to load and inspect `Cargo.toml` metadata. +//! +//! See `Manifest::from_slice`. + +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::io; +use std::io::Read; +use std::path::Path; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct I18nConfig { + #[serde(default = "default_locale")] + pub default_locale: String, + #[serde(default = "available_locales")] + pub available_locales: Vec, + #[serde(default = "load_path")] + pub load_path: String, + #[serde(default = "fallback")] + pub fallback: Vec, + #[serde(default = "minify_key")] + pub minify_key: bool, + #[serde(default = "minify_key_len")] + pub minify_key_len: usize, + #[serde(default = "minify_key_prefix")] + pub minify_key_prefix: String, + #[serde(default = "minify_key_thresh")] + pub minify_key_thresh: usize, +} + +impl I18nConfig { + pub fn new() -> Self { + Self { + default_locale: "en".to_string(), + available_locales: vec!["en".to_string()], + load_path: "./locales".to_string(), + fallback: vec![], + minify_key: crate::DEFAULT_MINIFY_KEY, + minify_key_len: crate::DEFAULT_MINIFY_KEY_LEN, + minify_key_prefix: crate::DEFAULT_MINIFY_KEY_PREFIX.to_string(), + minify_key_thresh: crate::DEFAULT_MINIFY_KEY_THRESH, + } + } + + pub fn load(cargo_root: &Path) -> io::Result { + let cargo_file = cargo_root.join("Cargo.toml"); + let mut file = fs::File::open(&cargo_file) + .unwrap_or_else(|e| panic!("Fail to open {}, {}", cargo_file.display(), e)); + + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + Self::parse(&contents) + } + + pub fn parse(contents: &str) -> io::Result { + if !contents.contains("[i18n]") && !contents.contains("[package.metadata.i18n]") { + return Ok(I18nConfig::default()); + } + let contents = contents.replace("[package.metadata.i18n]", "[i18n]"); + let mut config: MainConfig = toml::from_str(&contents) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + + // Push default_locale + config + .i18n + .available_locales + .insert(0, config.i18n.default_locale.clone()); + + // unqiue + config.i18n.available_locales = + config.i18n.available_locales.into_iter().unique().collect(); + + Ok(config.i18n) + } +} + +impl Default for I18nConfig { + fn default() -> Self { + Self::new() + } +} + +fn default_locale() -> String { + I18nConfig::default().default_locale +} + +fn available_locales() -> Vec { + I18nConfig::default().available_locales +} + +fn load_path() -> String { + I18nConfig::default().load_path +} + +fn fallback() -> Vec { + I18nConfig::default().fallback +} + +fn minify_key() -> bool { + I18nConfig::default().minify_key +} + +fn minify_key_len() -> usize { + I18nConfig::default().minify_key_len +} + +fn minify_key_prefix() -> String { + I18nConfig::default().minify_key_prefix +} + +fn minify_key_thresh() -> usize { + I18nConfig::default().minify_key_thresh +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub struct MainConfig { + pub i18n: I18nConfig, +} + +#[test] +fn test_parse() { + let contents = r#" + [i18n] + default-locale = "en" + available-locales = ["zh-CN"] + load-path = "./my-locales" + fallback = ["zh"] + minify-key = true + minify-key-len = 12 + minify-key-prefix = "T." + minify-key-thresh = 16 + "#; + + let cfg = I18nConfig::parse(contents).unwrap(); + assert_eq!(cfg.default_locale, "en"); + assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); + assert_eq!(cfg.load_path, "./my-locales"); + assert_eq!(cfg.fallback, vec!["zh"]); + assert_eq!(cfg.minify_key, true); + assert_eq!(cfg.minify_key_len, 12); + assert_eq!(cfg.minify_key_prefix, "T."); + assert_eq!(cfg.minify_key_thresh, 16); + + let contents = r#" + [i18n] + available-locales = ["zh-CN", "de", "de"] + load-path = "./my-locales" + "#; + let cfg = I18nConfig::parse(contents).unwrap(); + assert_eq!(cfg.default_locale, "en"); + assert_eq!(cfg.available_locales, vec!["en", "zh-CN", "de"]); + assert_eq!(cfg.load_path, "./my-locales"); + + let contents = ""; + let cfg = I18nConfig::parse(contents).unwrap(); + assert_eq!(cfg.default_locale, "en"); + assert_eq!(cfg.available_locales, vec!["en"]); + assert_eq!(cfg.load_path, "./locales"); +} + +#[test] +fn test_parse_with_metadata() { + let contents = r#" + [package.metadata.i18n] + default-locale = "en" + available-locales = ["zh-CN"] + load-path = "./my-locales" + fallback = ["zh"] + minify-key = true + minify-key-len = 12 + minify-key-prefix = "T." + minify-key-thresh = 16 + "#; + + let cfg = I18nConfig::parse(contents).unwrap(); + assert_eq!(cfg.default_locale, "en"); + assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); + assert_eq!(cfg.load_path, "./my-locales"); + assert_eq!(cfg.fallback, vec!["zh"]); + assert_eq!(cfg.minify_key, true); + assert_eq!(cfg.minify_key_len, 12); + assert_eq!(cfg.minify_key_prefix, "T."); + assert_eq!(cfg.minify_key_thresh, 16); +} + +#[test] +fn test_load_default() { + let workdir = Path::new(env!["CARGO_MANIFEST_DIR"]); + + let cfg = I18nConfig::load(workdir).unwrap(); + assert_eq!(cfg.default_locale, "en"); + assert_eq!(cfg.available_locales, vec!["en"]); + assert_eq!(cfg.load_path, "./locales"); +} + +#[test] +fn test_load() { + let workdir = Path::new(env!["CARGO_MANIFEST_DIR"]); + let cargo_root = workdir.join("../../examples/foo"); + + let cfg = I18nConfig::load(&cargo_root).unwrap(); + assert_eq!(cfg.default_locale, "en"); + assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); +} diff --git a/crates/support/src/cow_str.rs b/crates/support/src/cow_str.rs new file mode 100644 index 0000000..7b7fc44 --- /dev/null +++ b/crates/support/src/cow_str.rs @@ -0,0 +1,111 @@ +use std::borrow::Cow; +use std::sync::Arc; + +/// A wrapper for `Cow<'a, str>` that is specifically designed for use with the `tr!` macro. +/// +/// This wrapper provides additional functionality or optimizations when handling strings in the `tr!` macro. +pub struct CowStr<'a>(Cow<'a, str>); + +impl<'a> CowStr<'a> { + pub fn as_str(&self) -> &str { + self.0.as_ref() + } + + pub fn into_inner(self) -> Cow<'a, str> { + self.0 + } +} + +macro_rules! impl_convert_from_numeric { + ($typ:ty) => { + impl<'a> From<$typ> for CowStr<'a> { + fn from(val: $typ) -> Self { + Self(Cow::from(format!("{}", val))) + } + } + }; +} + +impl_convert_from_numeric!(i8); +impl_convert_from_numeric!(i16); +impl_convert_from_numeric!(i32); +impl_convert_from_numeric!(i64); +impl_convert_from_numeric!(i128); +impl_convert_from_numeric!(isize); + +impl_convert_from_numeric!(u8); +impl_convert_from_numeric!(u16); +impl_convert_from_numeric!(u32); +impl_convert_from_numeric!(u64); +impl_convert_from_numeric!(u128); +impl_convert_from_numeric!(usize); + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Arc) -> Self { + Self(Cow::Owned(s.to_string())) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Box) -> Self { + Self(Cow::Owned(s.to_string())) + } +} + +impl<'a> From<&'a str> for CowStr<'a> { + #[inline] + fn from(s: &'a str) -> Self { + Self(Cow::Borrowed(s)) + } +} + +impl<'a> From<&&'a str> for CowStr<'a> { + #[inline] + fn from(s: &&'a str) -> Self { + Self(Cow::Borrowed(s)) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Arc<&'a str>) -> Self { + Self(Cow::Borrowed(*s)) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Box<&'a str>) -> Self { + Self(Cow::Borrowed(*s)) + } +} + +impl<'a> From for CowStr<'a> { + #[inline] + fn from(s: String) -> Self { + Self(Cow::from(s)) + } +} + +impl<'a> From<&'a String> for CowStr<'a> { + #[inline] + fn from(s: &'a String) -> Self { + Self(Cow::Borrowed(s)) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Arc) -> Self { + Self(Cow::Owned(s.to_string())) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Box) -> Self { + Self(Cow::from(*s)) + } +} diff --git a/crates/support/src/lib.rs b/crates/support/src/lib.rs index a8fadd1..9dafd9d 100644 --- a/crates/support/src/lib.rs +++ b/crates/support/src/lib.rs @@ -5,8 +5,17 @@ use std::{collections::HashMap, path::Path}; mod atomic_str; mod backend; +mod config; +mod cow_str; +mod minify_key; pub use atomic_str::AtomicStr; pub use backend::{Backend, BackendExt, SimpleBackend}; +pub use config::I18nConfig; +pub use cow_str::CowStr; +pub use minify_key::{ + minify_key, MinifyKey, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, + DEFAULT_MINIFY_KEY_THRESH, +}; type Locale = String; type Value = serde_json::Value; @@ -96,7 +105,8 @@ pub fn load_locales bool>( .read_to_string(&mut content) .expect("Read file failed."); - let trs = parse_file(&content, ext, locale).expect("Parse file failed."); + let trs = parse_file(&content, ext, locale) + .expect(&format!("Parse file `{}` failed", entry.display())); trs.into_iter().for_each(|(k, new_value)| { translations @@ -132,11 +142,9 @@ fn parse_file(content: &str, ext: &str, locale: &str) -> Result { - return Ok(parse_file_v1(locale, &v)); + Err("Invalid locale file format, please check the version field".into()) } + _ => Ok(parse_file_v1(locale, &v)), }, Err(e) => Err(e), } @@ -150,7 +158,7 @@ fn parse_file(content: &str, ext: &str, locale: &str) -> Result Translations { - return Translations::from([(locale.to_string(), data.clone())]); + Translations::from([(locale.to_string(), data.clone())]) } /// Locale file format v2 @@ -205,7 +213,7 @@ fn parse_file_v2(key_prefix: &str, data: &serde_json::Value) -> Option), iter them and convert them and insert into trs let key = format_keys(&[&key_prefix, &key]); - if let Some(sub_trs) = parse_file_v2(&key, &value) { + if let Some(sub_trs) = parse_file_v2(&key, value) { // println!("--------------- sub_trs:\n{:?}", sub_trs); // Merge the sub_trs into trs for (locale, sub_value) in sub_trs { @@ -234,7 +242,7 @@ fn get_version(data: &serde_json::Value) -> usize { return version.as_u64().unwrap_or(1) as usize; } - return 1; + 1 } /// Join the keys with dot, if any key is empty, omit it. diff --git a/crates/support/src/minify_key.rs b/crates/support/src/minify_key.rs new file mode 100644 index 0000000..02475e6 --- /dev/null +++ b/crates/support/src/minify_key.rs @@ -0,0 +1,166 @@ +use once_cell::sync::Lazy; +use siphasher::sip128::SipHasher13; +use std::borrow::Cow; + +/// The default value of `minify_key` feature. +pub const DEFAULT_MINIFY_KEY: bool = false; + +/// The length of auto-generated translation key +pub const DEFAULT_MINIFY_KEY_LEN: usize = 24; + +/// The prefix of auto-generated translation key +pub const DEFAULT_MINIFY_KEY_PREFIX: &str = ""; + +/// The minimum length of the value to be generated the translation key +pub const DEFAULT_MINIFY_KEY_THRESH: usize = 127; + +// The hasher for generate the literal translation key +static TR_KEY_HASHER: Lazy = Lazy::new(SipHasher13::new); + +/// Calculate a 128-bit siphash of a value. +pub fn hash128 + ?Sized>(value: &T) -> u128 { + TR_KEY_HASHER.hash(value.as_ref()).as_u128() +} + +/// Generate a translation key from a value. +/// +/// # Arguments +/// +/// * `value` - The value to be generated. +/// * `key_len` - The length of the translation key. +/// * `prefix` - The prefix of the translation key. +/// * `threshold` - The minimum length of the value to be generated. +/// +/// # Returns +/// +/// * If `value.len() <= threshold` then returns the origin value. +/// * Otherwise, returns a base62 encoded 128 bits hashed translation key. +/// +pub fn minify_key<'r>( + value: &'r str, + key_len: usize, + prefix: &str, + threshold: usize, +) -> Cow<'r, str> { + if value.len() <= threshold { + return Cow::Borrowed(value); + } + let encoded = base62::encode(hash128(value)); + let key_len = key_len.min(encoded.len()); + format!("{}{}", prefix, &encoded[..key_len]).into() +} + +/// A trait for generating translation key from a value. +pub trait MinifyKey<'a> { + /// Generate translation key from a value. + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str>; +} + +impl<'a> MinifyKey<'a> for str { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for &str { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for String { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::Borrowed(self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for &String { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::from(*self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for Cow<'a, str> { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::Borrowed(self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for &Cow<'a, str> { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::Borrowed(*self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_minify_key() { + let msg = "Hello, world!"; + assert_eq!( + minify_key(msg, 24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + let msg = format!("Hello, world!"); + assert_eq!( + minify_key(&msg, 24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 128), + "Hello, world!" + ); + let msg = &msg; + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + let msg = Cow::Owned("Hello, world!".to_owned()); + assert_eq!( + minify_key(&msg, 24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 128), + "Hello, world!" + ); + assert_eq!("".minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), ""); + assert_eq!( + "1".minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "knx7vOJBRfzgQvNfEkbEi" + ); + assert_eq!("1".minify_key(24, "t_", 0), "t_knx7vOJBRfzgQvNfEkbEi"); + } +} diff --git a/examples/app-egui/Cargo.toml b/examples/app-egui/Cargo.toml new file mode 100644 index 0000000..97a1a73 --- /dev/null +++ b/examples/app-egui/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "app-egui" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +eframe = "0.25.0" +env_logger = { version = "0.11", optional = true } +log = { version = "0.4", optional = true } +rust-i18n = { path = "../.." } + +[features] +log-miss-tr = ["env_logger", "log", "rust-i18n/log-miss-tr"] + +[package.metadata.i18n] +available-locales = ["en", "fr", "ja", "ko", "zh", "zh-CN"] +default-locale = "en" +minify-key = true +minify-key-len = 12 +minify-key-prefix = "T." +minify-key-thresh = 8 diff --git a/examples/app-egui/locales/v2.yml b/examples/app-egui/locales/v2.yml new file mode 100644 index 0000000..3e37ef0 --- /dev/null +++ b/examples/app-egui/locales/v2.yml @@ -0,0 +1,52 @@ +_version: 2 + +Arthur: + en: Arthur + fr: Arthur + ja: アーサー + ko: 아서 + zh: 亚瑟 + zh-TW: 亞瑟 +T.3UjfzBSnm1Qc: + en: My egui App + fr: Mon application egui + ja: 私の EGUI アプリ + ko: 내 egui 앱 + zh: 我的 EGUI 应用 + zh-TW: 我的 EGUI 應用 +T.4E2YxmZoydHX: + en: Click each year + fr: Cliquez sur chaque année + ja: 各年をクリック + ko: 각 연도를 클릭 + zh: 点击大一岁 + zh-TW: 點擊大一歲 +T.5o7tEO3Df7z8: + en: "Hello '%{name}', age %{age}" + fr: "Bonjour '%{name}', âge %{age}" + ja: "こんにちは '%{name}', 年齢 %{age}" + ko: "안녕하세요 '%{name}', 나이 %{age}" + zh: "你好 “%{name}”, 年龄 %{age}" + zh-TW: "妳好 “%{name}”, 年齡 %{age}" +T.6JOKobVbFAxv: + en: My egui Application + fr: Mon application egui + ja: 私の EGUI アプリケーション + ko: 내 egui 애플리케이션 + zh: 我的 EGUI 应用程序 + zh-TW: 我的 EGUI 應用程式 +T.gOXpNwPJI1fQ: + en: "Your name: " + fr: "Votre nom : " + ja: "あなたの名前：" + ko: "당신의 이름: " + zh: "你的名字：" + zh-TW: "你的名字：" +age: + en: age + fr: âge + ja: 年齢 + ko: 나이 + zh: 岁 + zh-TW: 歲 + diff --git a/examples/app-egui/src/main.rs b/examples/app-egui/src/main.rs new file mode 100644 index 0000000..3c5b9a9 --- /dev/null +++ b/examples/app-egui/src/main.rs @@ -0,0 +1,143 @@ +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] // hide console window on Windows in release + +use eframe::egui::{self, TextBuffer}; +use rust_i18n::t; + +rust_i18n::i18n!( + "locales", + minify_key = true, + minify_key_len = 12, + minify_key_prefix = "T.", + minify_key_thresh = 8 +); + +fn main() -> Result<(), eframe::Error> { + #[cfg(feature = "log-miss-tr")] + env_logger::init(); // Log to stderr (if you run with `RUST_LOG=debug`). + let options = eframe::NativeOptions { + viewport: egui::ViewportBuilder::default().with_inner_size([320.0, 240.0]), + ..Default::default() + }; + eframe::run_native( + t!("My egui App").as_str(), + options, + Box::new(|cc| { + // This gives us image support: + // egui_extras::install_image_loaders(&cc.egui_ctx); + let _ = setup_custom_fonts(&cc.egui_ctx); + Box::::default() + }), + ) +} + +struct MyApp { + name: String, + age: u32, + locale_id: usize, +} + +impl Default for MyApp { + fn default() -> Self { + Self { + name: t!("Arthur").into(), + age: 42, + locale_id: 0, + } + } +} + +impl eframe::App for MyApp { + fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) { + egui::CentralPanel::default().show(ctx, |ui| { + ui.heading(t!("My egui Application")); + ui.horizontal(|ui| { + let name_label = ui.label(t!("Your name: ")); + ui.text_edit_singleline(&mut self.name) + .labelled_by(name_label.id); + }); + ui.add(egui::Slider::new(&mut self.age, 0..=120).text(t!("age"))); + if ui.button(t!("Click each year")).clicked() { + self.age += 1; + } + ui.label(t!("Hello '%{name}', age %{age}", name => self.name, age => self.age)); + + ui.separator(); + + ui.horizontal(|ui| { + let locales = rust_i18n::available_locales!(); + for (i, locale) in locales.iter().enumerate() { + if ui + .selectable_value(&mut self.locale_id, i, *locale) + .changed() + { + rust_i18n::set_locale(locale); + ui.ctx().send_viewport_cmd(egui::ViewportCommand::Title( + t!("My egui App").to_string(), + )); + } + } + }); + }); + } +} + +#[cfg(windows)] +fn try_load_system_font() -> Result, std::io::Error> { + let font_files = &[ + "C:/Windows/Fonts/msyh.ttc", + "C:/Windows/Fonts/msjh.ttf", + "C:/Windows/Fonts/yugothr.ttc", + "C:/Windows/Fonts/malgun.ttf", + ]; + + for font in font_files { + if let Ok(font) = std::fs::read(font) { + return Ok(font); + } + } + + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "No system font found", + )) +} + +#[cfg(windows)] +fn setup_custom_fonts(ctx: &egui::Context) -> Result<(), std::io::Error> { + // Start with the default fonts (we will be adding to them rather than replacing them). + let mut fonts = egui::FontDefinitions::default(); + + // Install my own font (maybe supporting non-latin characters). + // .ttf and .otf files supported. + let font_bytes = try_load_system_font()?; + fonts + .font_data + .insert("my_font".to_owned(), egui::FontData::from_owned(font_bytes)); + + // Put my font first (highest priority) for proportional text: + fonts + .families + .entry(egui::FontFamily::Proportional) + .or_default() + .insert(0, "my_font".to_owned()); + + // Put my font as last fallback for monospace: + fonts + .families + .entry(egui::FontFamily::Monospace) + .or_default() + .push("my_font".to_owned()); + + // Tell egui to use these fonts: + ctx.set_fonts(fonts); + + Ok(()) +} + +#[cfg(not(windows))] +fn setup_custom_fonts(ctx: &egui::Context) -> Result<(), std::io::Error> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Custom fonts not supported on this platform", + )) +} diff --git a/examples/app-metadata/Cargo.toml b/examples/app-metadata/Cargo.toml new file mode 100644 index 0000000..d6367c6 --- /dev/null +++ b/examples/app-metadata/Cargo.toml @@ -0,0 +1,18 @@ +[package] +edition = "2021" +name = "app-metadata" +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rust-i18n = { path = "../.." } + +[package.metadata.i18n] +available-locales = ["en", "zh"] +default-locale = "zh" +load-path = "locales" +minify-key = true +minify-key-len = 12 +minify-key-prefix = "T." +minify-key-thresh = 4 diff --git a/examples/app-metadata/locales/v2.yml b/examples/app-metadata/locales/v2.yml new file mode 100644 index 0000000..4a2b5e8 --- /dev/null +++ b/examples/app-metadata/locales/v2.yml @@ -0,0 +1,23 @@ +_version: 2 + +T.1LokVzuiIrh1: + en: "Hello, world!" + zh: "你好，世界！" +T.53pFZEJAcwid: + en: ABCDEF + zh: 甲乙丙丁戊己 +T.6zJ2nRuJ42Z5: + en: ABCDE + zh: 甲乙丙丁戊 +a: + en: A + zh: 甲 +ab: + en: AB + zh: 甲乙 +abc: + en: ABC + zh: 甲乙丙 +abcd: + en: ABCD + zh: 甲乙丙丁 diff --git a/examples/app-metadata/src/main.rs b/examples/app-metadata/src/main.rs new file mode 100644 index 0000000..b46d066 --- /dev/null +++ b/examples/app-metadata/src/main.rs @@ -0,0 +1,24 @@ +use rust_i18n::t; + +rust_i18n::i18n!(metadata = true); + +fn main() { + let locales = rust_i18n::available_locales!(); + println!("Available locales: {:?}", locales); + println!(); + + assert_eq!(t!("a"), "甲"); + assert_eq!(t!("ab"), "甲乙"); + assert_eq!(t!("abc"), "甲乙丙"); + assert_eq!(t!("abcd"), "甲乙丙丁"); + assert_eq!(t!("abcde"), "甲乙丙丁戊"); + assert_eq!(t!("abcdef"), "甲乙丙丁戊己"); + assert_eq!(t!("Hello, world!"), "你好，世界！"); + assert_eq!(t!("a", locale = "en"), "A"); + assert_eq!(t!("ab", locale = "en"), "AB"); + assert_eq!(t!("abc", locale = "en"), "ABC"); + assert_eq!(t!("abcd", locale = "en"), "ABCD"); + assert_eq!(t!("abcde", locale = "en"), "ABCDE"); + assert_eq!(t!("abcdef", locale = "en"), "ABCDEF"); + assert_eq!(t!("Hello, world!", locale = "en"), "Hello, world!"); +} diff --git a/examples/app-minify-key/Cargo.toml b/examples/app-minify-key/Cargo.toml new file mode 100644 index 0000000..4ef9c15 --- /dev/null +++ b/examples/app-minify-key/Cargo.toml @@ -0,0 +1,18 @@ +[package] +edition = "2021" +name = "app-minify-key" +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +env_logger = { version = "0.11", optional = true } +log = { version = "0.4", optional = true } +rust-i18n = { path = "../.." } + +[features] +log-miss-tr = ["env_logger", "log", "rust-i18n/log-miss-tr"] + +[package.metadata.i18n] +available-locales = ["en", "zh-CN"] +default-locale = "en" diff --git a/examples/app-minify-key/locales/v2.yml b/examples/app-minify-key/locales/v2.yml new file mode 100644 index 0000000..db97dfd --- /dev/null +++ b/examples/app-minify-key/locales/v2.yml @@ -0,0 +1,78 @@ +_version: 2 +T.29xGXAUPAkgvVzCf9ES3q8: + en: Apple + de: Apfel + fr: Pomme + it: Mela + ja: りんご + ko: 사과 + ru: Яблоко + vi: Táo + zh: 苹果 + zh-TW: 蘋果 +T.7MQVq9vgi0h6pLE47CdWXH: + en: Banana + de: Banane + fr: Banane + it: Banana + ja: バナナ + ko: 바나나 + ru: Банан + vi: Chuối + zh: 香蕉 + zh-TW: 香蕉 +T.2RohljPx99sA18L8E5oTD4: + en: Orange + de: Orange + fr: Orange + it: Arancia + ja: オレンジ + ko: 오렌지 + ru: Апельсин + vi: Cam + zh: 橘子 + zh-TW: 橘子 +T.KtVLkBzuyfHoJZtCWEIOU: + en: Hello + de: Hallo + fr: Bonjour + it: Ciao + ja: こんにちは + ko: 안녕하세요 + ru: Привет + vi: Xin chào + zh: 你好 + zh-TW: 妳好 +T.tvpNzQjFwc0trHvgzSBxX: + en: Hello, %{name}! + de: Hallo, %{name}! + fr: Bonjour, %{name}! + it: Ciao, %{name}! + ja: こんにちは、%{name}！ + ko: 안녕하세요, %{name}! + ru: Привет, %{name}! + vi: Xin chào, %{name}! + zh: 你好，%{name}！ + zh-TW: 妳好，%{name}！ +T.kmFrQ2nnJsvUh3Ckxmki0: + en: "Hello, %{name}. Your message is: %{msg}" + de: "Hallo, %{name}. Deine Nachricht ist: %{msg}" + fr: "Bonjour, %{name}. Votre message est: %{msg}" + it: "Ciao, %{name}. Il tuo messaggio è: %{msg}" + ja: "こんにちは、%{name}。あなたのメッセージは: %{msg}" + ko: "안녕하세요, %{name}. 당신의 메시지는: %{msg}" + ru: "Привет, %{name}. Ваше сообщение: %{msg}" + vi: "Xin chào, %{name}. Tin nhắn của bạn là: %{msg}" + zh: "你好，%{name}。这是你的消息：%{msg}" + zh-TW: "妳好，%{name}。這是妳的消息：%{msg}" +T.7hWbTwvMpr0H0oDLIQlfrm: + en: You have %{count} messages. + de: Du hast %{count} Nachrichten. + fr: Vous avez %{count} messages. + it: Hai %{count} messaggi. + ja: あなたは %{count} 件のメッセージを持っています。 + ko: 당신은 %{count} 개의 메시지를 가지고 있습니다. + ru: У вас %{count} сообщений. + vi: Bạn có %{count} tin nhắn. + zh: 你收到了 %{count} 条新消息。 + zh-TW: 妳收到了 %{count} 條新消息。 diff --git a/examples/app-minify-key/src/main.rs b/examples/app-minify-key/src/main.rs new file mode 100644 index 0000000..2b714e8 --- /dev/null +++ b/examples/app-minify-key/src/main.rs @@ -0,0 +1,87 @@ +use rust_i18n::t; + +rust_i18n::i18n!( + "locales", + minify_key = true, + minify_key_len = 24, + minify_key_prefix = "T.", + minify_key_thresh = 4 +); + +#[cfg(feature = "log-miss-tr")] +fn set_logger() { + env_logger::builder() + .filter_level(log::LevelFilter::Debug) + .parse_default_env() + .init(); +} + +#[cfg(not(feature = "log-miss-tr"))] +fn set_logger() {} + +fn main() { + set_logger(); + + let locales = rust_i18n::available_locales!(); + println!("Available locales: {:?}", locales); + println!(); + + println!("Translation of string literals:"); + for locale in &locales { + println!( + "{:>8} => {} ({})", + "Hello", + t!("Hello", locale = locale), + locale + ); + } + println!(); + + println!("Translation of string literals with patterns:"); + for locale in &locales { + println!( + "Hello, %{{name}}! => {} ({})", + t!("Hello, %{name}!", name = "World", locale = locale), + locale + ); + } + println!(); + + println!("Translation of string literals with specified arguments:"); + for i in (0..10000).step_by(50) { + println!( + "Zero padded number: %{{count}} => {}", + t!("Zero padded number: %{count}", count = i : {:08}), + ); + } + println!(); + + println!("Handling of missing translations:"); + for locale in &locales { + println!( + "{:>8} => {} ({locale})", + "The message is untranslated!", + t!("The message is untranslated!", locale = locale) + ); + } + println!(); + + println!("Translation of runtime strings:"); + let src_list = ["Apple", "Banana", "Orange"]; + for src in src_list.iter() { + for locale in &locales { + let translated = t!(*src, locale = locale); + println!("{:>8} => {} ({locale})", src, translated); + } + } + println!(); + + if cfg!(feature = "log-miss-tr") { + println!("Translates runtime strings and logs when a lookup is missing:"); + for locale in &locales { + let msg = "Foo Bar".to_string(); + println!("{:>8} => {} ({locale})", &msg, t!(&msg, locale = locale)); + } + println!(); + } +} diff --git a/src/lib.rs b/src/lib.rs index 59ab8c2..b0f67d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,8 +6,11 @@ use once_cell::sync::Lazy; #[doc(hidden)] pub use once_cell; -pub use rust_i18n_macro::{i18n, key}; -pub use rust_i18n_support::{AtomicStr, Backend, BackendExt, SimpleBackend}; +pub use rust_i18n_macro::{i18n, mikey, tr, vakey}; +pub use rust_i18n_support::{ + AtomicStr, Backend, BackendExt, CowStr, MinifyKey, SimpleBackend, DEFAULT_MINIFY_KEY, + DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, DEFAULT_MINIFY_KEY_THRESH, +}; static CURRENT_LOCALE: Lazy = Lazy::new(|| AtomicStr::from("en")); @@ -89,71 +92,96 @@ pub fn replace_patterns(input: &str, patterns: &[&str], values: &[String]) -> St /// Get I18n text /// +/// This macro forwards to the `crate::_rust_i18n_t!` macro, which is generated by the [`i18n!`] macro. +/// +/// # Arguments +/// +/// * `expr` - The key or message for translation. +/// - A key usually looks like `"foo.bar.baz"`. +/// - A literal message usually looks like `"Hello, world!"`. +/// - The variable names in the message should be wrapped in `%{}`, like `"Hello, %{name}!"`. +/// - Dynamic messages are also supported, such as `t!(format!("Hello, {}!", name))`. +/// However, if `minify_key` is enabled, the entire message will be hashed and used as a key for every lookup, which may consume more CPU cycles. +/// * `locale` - The locale to use. If not specified, the current locale will be used. +/// * `args` - The arguments to be replaced in the translated text. +/// - These should be passed in the format `key = value` or `key => value`. +/// - Alternatively, you can specify the value format using the `key = value : {:format_specifier}` syntax. +/// For example, `key = value : {:08}` will format the value as a zero-padded string with a length of 8. +/// +/// # Example +/// /// ```no_run /// #[macro_use] extern crate rust_i18n; -/// # fn _rust_i18n_translate(locale: &str, key: &str) -> String { todo!() } +/// +/// # macro_rules! t { ($($all:tt)*) => {} } /// # fn main() { /// // Simple get text with current locale -/// t!("greeting"); // greeting: "Hello world" => "Hello world" +/// t!("greeting"); +/// // greeting: "Hello world" => "Hello world" +/// /// // Get a special locale's text -/// t!("greeting", locale = "de"); // greeting: "Hallo Welt!" => "Hallo Welt!" +/// t!("greeting", locale = "de"); +/// // greeting: "Hallo Welt!" => "Hallo Welt!" /// /// // With variables -/// t!("messages.hello", name = "world"); // messages.hello: "Hello, {name}" => "Hello, world" -/// t!("messages.foo", name = "Foo", other ="Bar"); // messages.foo: "Hello, {name} and {other}" => "Hello, Foo and Bar" +/// t!("messages.hello", name = "world"); +/// // messages.hello: "Hello, %{name}" => "Hello, world" +/// t!("messages.foo", name = "Foo", other ="Bar"); +/// // messages.foo: "Hello, %{name} and %{other}" => "Hello, Foo and Bar" +/// +/// // With variables and format specifiers +/// t!("Hello, %{name}, you serial number is: %{sn}", name = "Jason", sn = 123 : {:08}); +/// // => "Hello, Jason, you serial number is: 000000123" /// /// // With locale and variables -/// t!("messages.hello", locale = "de", name = "Jason"); // messages.hello: "Hallo, {name}" => "Hallo, Jason" +/// t!("messages.hello", locale = "de", name = "Jason"); +/// // messages.hello: "Hallo, %{name}" => "Hallo, Jason" /// # } /// ``` #[macro_export] #[allow(clippy::crate_in_macro_def)] macro_rules! t { - // t!("foo") - ($key:expr) => { - crate::_rust_i18n_translate(&rust_i18n::locale(), $key) - }; - - // t!("foo", locale = "en") - ($key:expr, locale = $locale:expr) => { - crate::_rust_i18n_translate($locale, $key) - }; - - // t!("foo", locale = "en", a = 1, b = "Foo") - ($key:expr, locale = $locale:expr, $($var_name:tt = $var_val:expr),+ $(,)?) => { - { - let message = crate::_rust_i18n_translate($locale, $key); - let patterns: &[&str] = &[ - $(rust_i18n::key!($var_name)),+ - ]; - let values = &[ - $(format!("{}", $var_val)),+ - ]; - - let output = rust_i18n::replace_patterns(message.as_ref(), patterns, values); - std::borrow::Cow::from(output) - } - }; - - // t!("foo %{a} %{b}", a = "bar", b = "baz") - ($key:expr, $($var_name:tt = $var_val:expr),+ $(,)?) => { - { - t!($key, locale = &rust_i18n::locale(), $($var_name = $var_val),*) - } - }; - - // t!("foo %{a} %{b}", locale = "en", "a" => "bar", "b" => "baz") - ($key:expr, locale = $locale:expr, $($var_name:tt => $var_val:expr),+ $(,)?) => { - { - t!($key, locale = $locale, $($var_name = $var_val),*) - } - }; + ($($all:tt)*) => { + crate::_rust_i18n_t!($($all)*) + } +} - // t!("foo %{a} %{b}", "a" => "bar", "b" => "baz") - ($key:expr, $($var_name:tt => $var_val:expr),+ $(,)?) => { - { - t!($key, locale = &rust_i18n::locale(), $($var_name = $var_val),*) - } +/// A macro that generates a translation key and corresponding value pair from a given input value. +/// +/// It's useful when you want to use a long string as a key, but you don't want to type it twice. +/// +/// # Arguments +/// +/// * `msg` - The input value. +/// +/// # Returns +/// +/// A tuple of `(key, msg)`. +/// +/// # Example +/// +/// ```no_run +/// use rust_i18n::{t, tkv}; +/// +/// # macro_rules! t { ($($all:tt)*) => { } } +/// # macro_rules! tkv { ($($all:tt)*) => { (1,2) } } +/// +/// let (key, msg) = tkv!( +/// r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. +/// +/// Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# +/// ); +/// // Use the `key` and `msg` in `t!` macro +/// t!((key, msg), bar = "foo"); +/// // ... +/// // In other parts of the code, you can reuse the same `key` and `msg` +/// t!((key, msg), bar = "baz"); +/// ``` +#[macro_export] +#[allow(clippy::crate_in_macro_def)] +macro_rules! tkv { + ($msg:literal) => { + crate::_rust_i18n_tkv!($msg) }; } diff --git a/tests/i18n_minify_key.rs b/tests/i18n_minify_key.rs new file mode 100644 index 0000000..27c8445 --- /dev/null +++ b/tests/i18n_minify_key.rs @@ -0,0 +1,73 @@ +rust_i18n::i18n!( + "./tests/locales", + fallback = "en", + minify_key = true, + minify_key_len = 24, + minify_key_prefix = "tr_", + minify_key_thresh = 4 +); + +#[cfg(test)] +mod tests { + use rust_i18n::{t, tkv}; + + #[test] + fn test_i18n_attrs() { + assert_eq!(crate::_RUST_I18N_MINIFY_KEY, true); + assert_eq!(crate::_RUST_I18N_MINIFY_KEY_LEN, 24); + assert_eq!(crate::_RUST_I18N_MINIFY_KEY_PREFIX, "tr_"); + assert_eq!(crate::_RUST_I18N_MINIFY_KEY_THRESH, 4); + } + + #[test] + fn test_t() { + assert_eq!(t!("Bar - Hello, World!"), "Bar - Hello, World!"); + assert_eq!( + t!("Bar - Hello, World!", locale = "en"), + "Bar - Hello, World!" + ); + assert_eq!( + t!("Bar - Hello, World!", locale = "zh-CN"), + "Bar - 你好世界！" + ); + let fruits = vec!["Apple", "Banana", "Orange"]; + let fruits_translated = vec!["苹果", "香蕉", "橘子"]; + for (src, dst) in fruits.iter().zip(fruits_translated.iter()) { + assert_eq!(t!(*src, locale = "zh-CN"), *dst); + } + let msg = "aka".to_string(); + let i = 0; + assert_eq!(t!(msg, name => & i : {} ), "aka"); + assert_eq!(t!("hello"), "hello"); + assert_eq!(t!("hello",), "hello"); + assert_eq!(t!("hello", locale = "en"), "hello"); + assert_eq!(t!(format!("hello"), locale = "en"), "hello"); + assert_eq!(t!("Hello, %{name}", name = "Bar"), "Hello, Bar"); + assert_eq!( + t!("You have %{count} messages.", locale = "zh-CN", count = 1 + 2,,,), + "你收到了 3 条新消息。" + ); + let (key, msg) = tkv!( + r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. + + Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# + ); + assert_eq!(t!((key, msg)).as_ptr(), msg.as_ptr()); + assert_eq!(t!((key, msg), locale = "en"), msg); + assert_eq!(t!((key, msg), locale = "de"), msg); + assert_eq!(t!((key, msg), locale = "zh"), msg); + } + + #[test] + fn test_tkv() { + let (key, msg) = tkv!(""); + assert_eq!(key, ""); + assert_eq!(msg, ""); + let (key, msg) = tkv!("Hey"); + assert_eq!(key, "Hey"); + assert_eq!(msg, "Hey"); + let (key, msg) = tkv!("Hello, world!"); + assert_eq!(key, "tr_1LokVzuiIrh1xByyZG4wjZ"); + assert_eq!(msg, "Hello, world!"); + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 33e8565..8eb3e07 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -84,6 +84,10 @@ mod tests { } } + mod test5 { + rust_i18n::i18n!(metadata = true); + } + #[test] fn check_test_environment() { assert_eq!( @@ -114,7 +118,7 @@ mod tests { fn test_available_locales() { assert_eq!( rust_i18n::available_locales!(), - &["en", "ja", "pt", "zh", "zh-CN"] + &["de", "en", "fr", "ja", "ko", "pt", "ru", "vi", "zh", "zh-CN"] ); } diff --git a/tests/locales/v2.yml b/tests/locales/v2.yml index f32d082..0be5e12 100644 --- a/tests/locales/v2.yml +++ b/tests/locales/v2.yml @@ -11,3 +11,84 @@ nested_locale_test: en: "Hello test3" ja: "こんにちは test3" zh-CN: "你好 test3" +tr_1bHAL18drdyculzJ6OdjT0: + en: "Hello, you id is: 123" + de: "Hallo, deine ID ist: 123" + fr: "Bonjour, votre ID est : 123" + ja: "こんにちは、あなたのIDは 123 です" + ko: "안녕하세요, 당신의 ID는 123입니다" + ru: "Привет, ваш ID: 123" + vi: "Xin chào, ID của bạn là: 123" + zh-CN: "你好，你的 ID 是：123" +tr_29xGXAUPAkgvVzCf9ES3q8: + en: Apple + de: Apfel + fr: Pomme + ja: りんご + ko: 사과 + ru: Яблоко + vi: Táo + zh-CN: 苹果 +tr_2xBwKSjL3poKDdxU7Mej0e: + en: Bar - Hello, World! + de: Bar - Hallo, Welt! + fr: Bar - Bonjour, monde! + ja: Bar - こんにちは世界！ + ko: Bar - 안녕하세요, 세계여! + ru: Bar - Привет, мир! + vi: Bar - Xin chào, thế giới! + zh-CN: Bar - 你好世界！ +tr_2RohljPx99sA18L8E5oTD4: + en: Orange + de: Orange + fr: Orange + ja: オレンジ + ko: 오렌지 + ru: Апельсин + vi: Cam + zh-CN: 橘子 +tr_7hWbTwvMpr0H0oDLIQlfrm: + en: You have %{count} messages. + de: Du hast %{count} Nachrichten. + fr: Vous avez %{count} messages. + ja: あなたは %{count} 件のメッセージを持っています。 + ko: 당신은 %{count} 개의 메시지를 가지고 있습니다. + ru: У вас %{count} сообщений. + vi: Bạn có %{count} tin nhắn. + zh-CN: 你收到了 %{count} 条新消息。 +tr_7MQVq9vgi0h6pLE47CdWXH: + en: Banana + de: Banane + fr: Banane + ja: バナナ + ko: 바나나 + ru: Банан + vi: Chuối + zh-CN: 香蕉 +tr_kmFrQ2nnJsvUh3Ckxmki0: + en: "Hello, %{name}. Your message is: %{msg}" + de: "Hallo, %{name}. Deine Nachricht ist: %{msg}" + fr: "Bonjour, %{name}. Votre message est: %{msg}" + ja: "こんにちは、%{name}。あなたのメッセージは: %{msg}" + ko: "안녕하세요, %{name}. 당신의 메시지는: %{msg}" + ru: "Привет, %{name}. Ваше сообщение: %{msg}" + vi: "Xin chào, %{name}. Tin nhắn của bạn là: %{msg}" + zh-CN: "你好，%{name}。这是你的消息：%{msg}" +tr_tvpNzQjFwc0trHvgzSBxX: + en: Hello, %{name}! + de: Hallo, %{name}! + fr: Bonjour, %{name}! + ja: こんにちは、%{name}！ + ko: 안녕하세요, %{name}! + ru: Привет, %{name}! + vi: Xin chào, %{name}! + zh-CN: 你好，%{name}！ +tr_3w1UK0UDyZZMkgsPduhmOv: + en: "Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. \r\n You live in %{city} %{zip}. \r\n Your website is %{website}." + de: "Hallo %{name} %{surname}, deine Kontonummer ist %{id}, E-Mail-Adresse ist %{email}. \r\n Du lebst in %{city} %{zip}. \r\n Deine Website ist %{website}." + fr: "Bonjour %{name} %{surname}, votre identifiant de compte est %{id}, votre adresse e-mail est %{email}. \r\n Vous vivez à %{city} %{zip}. \r\n Votre site Web est %{website}." + ja: "こんにちは %{name} %{surname}、あなたのアカウントIDは %{id}、メールアドレスは %{email} です。 \r\n あなたは %{city} %{zip} に住んでいます。 \r\n あなたのウェブサイトは %{website} です。" + ko: "안녕하세요 %{name} %{surname}, 당신의 계정 ID는 %{id}, 이메일 주소는 %{email} 입니다. \r\n 당신은 %{city} %{zip} 에 살고 있습니다. \r\n 당신의 웹사이트는 %{website} 입니다." + ru: "Привет %{name} %{surname}, ваш ID аккаунта %{id}, адрес электронной почты %{email}. \r\n Вы живете в %{city} %{zip}. \r\n Ваш сайт %{website}." + vi: "Xin chào %{name} %{surname}, ID tài khoản của bạn là %{id}, địa chỉ email là %{email}. \r\n Bạn sống ở %{city} %{zip}. \r\n Trang web của bạn là %{website}." + zh-CN: "你好 %{name} %{surname}，你的帐户 ID 是 %{id}，电子邮件地址是 %{email}。 \r\n 你住在 %{city} %{zip}。 \r\n 你的网站是 %{website}。" diff --git a/tests/multi_threading.rs b/tests/multi_threading.rs index 7d55b52..3afb96d 100644 --- a/tests/multi_threading.rs +++ b/tests/multi_threading.rs @@ -1,9 +1,8 @@ +use rust_i18n::{set_locale, t}; use std::ops::Add; use std::thread::spawn; use std::time::{Duration, Instant}; -use rust_i18n::{set_locale, t}; - rust_i18n::i18n!("locales", fallback = "en"); #[test] @@ -32,3 +31,43 @@ fn test_load_and_store() { store.join().unwrap(); load.join().unwrap(); } + +#[test] +fn test_t_concurrent() { + let end = Instant::now().add(Duration::from_secs(3)); + let store = spawn(move || { + let mut i = 0u32; + while Instant::now() < end { + for _ in 0..100 { + i = i.wrapping_add(1); + if i % 2 == 0 { + set_locale(&format!("en-{i}")); + } else { + set_locale(&format!("fr-{i}")); + } + } + } + }); + let tasks: Vec<_> = (0..4) + .map(|_| { + spawn(move || { + let locales = rust_i18n::available_locales!(); + let num_locales = locales.len(); + while Instant::now() < end { + for i in 0..100usize { + let m = i.checked_rem(num_locales).unwrap_or_default(); + if m == 0 { + t!("hello"); + } else { + t!("hello", locale = locales[m]); + } + } + } + }) + }) + .collect(); + store.join().unwrap(); + for task in tasks { + task.join().unwrap(); + } +}