Explicit LocaleFallbacker argument for DatagenDriver

components/icu/Cargo.toml

@@ -35,6 +35,10 @@ icu_segmenter = { workspace = true }
 icu_timezone = { workspace = true }
 icu_experimental = { workspace = true, optional = true }
 
+# For markers_from_bin
+icu_registry = { workspace = true, optional = true }
+memchr = { workspace = true, optional = true }
+
 # For docs links and features
 icu_provider = { workspace = true }
 
@@ -108,6 +112,9 @@ compiled_data = [
     "icu_experimental?/compiled_data",
 ]
 datagen = [
+    "std",
+    "dep:icu_registry",
+    "dep:memchr",
     "icu_calendar/datagen",
     "icu_casemap/datagen",
     "icu_collator/datagen",

components/icu/src/datagen.rs

@@ -0,0 +1,106 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use icu_provider::prelude::*;
+use std::collections::HashSet;
+use std::path::Path;
+
+macro_rules! cb {
+    ($($marker:path = $path:literal,)+ #[experimental] $($emarker:path = $epath:literal,)+) => {
+        fn markers_for_bin_inner(bytes: &[u8]) -> Result<HashSet<DataMarkerInfo>, DataError> {
+            use std::sync::OnceLock;
+            use crate as icu;
+            static LOOKUP: OnceLock<std::collections::HashMap<&'static str, Option<DataMarkerInfo>>> = OnceLock::new();
+            let lookup = LOOKUP.get_or_init(|| {
+                [
+                    ("core/helloworld@1", Some(icu_provider::hello_world::HelloWorldV1Marker::INFO)),
+                    $(
+                        ($path, Some(<$marker>::INFO)),
+                    )+
+                    $(
+                        #[cfg(feature = "experimental")]
+                        ($epath, Some(<$emarker>::INFO)),
+                        #[cfg(not(feature = "experimental"))]
+                        ($epath, None),
+                    )+
+
+                ]
+                .into_iter()
+                .collect()
+            });
+
+            use memchr::memmem::*;
+
+            const LEADING_TAG: &[u8] = icu_provider::leading_tag!().as_bytes();
+            const TRAILING_TAG: &[u8] = icu_provider::trailing_tag!().as_bytes();
+
+            let trailing_tag = Finder::new(TRAILING_TAG);
+
+            find_iter(bytes, LEADING_TAG)
+                .map(|tag_position| tag_position + LEADING_TAG.len())
+                .filter_map(|marker_start| bytes.get(marker_start..))
+                .filter_map(move |marker_fragment| {
+                    trailing_tag
+                        .find(marker_fragment)
+                        .and_then(|end| marker_fragment.get(..end))
+                })
+                .map(std::str::from_utf8)
+                .filter_map(Result::ok)
+                .filter_map(|p| {
+                    match lookup.get(p) {
+                        Some(Some(marker)) => Some(Ok(*marker)),
+                        Some(None) => Some(Err(DataError::custom("This marker requires the `experimental` Cargo feature").with_display_context(p))),
+                        None => None,
+                    }
+                })
+                .collect::<Result<_, _>>()
+        }
+    }
+}
+icu_registry::registry!(cb);
+
+/// Parses a compiled binary and returns a list of [`DataMarkerInfo`]s that it uses *at runtime*.
+///
+/// This function is intended to be used for binaries that use `AnyProvider` or `BufferProvider`,
+/// for which the compiler cannot remove unused data. Using this function on a binary that only
+/// uses compiled data (through the `compiled_data` feature or manual baked data) will not return
+/// any markers, as the markers only exist in the type system.
+///
+/// # Example
+///
+/// ```no_run
+/// # use icu_provider::DataMarker;
+/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// assert_eq!(
+///     icu::markers_for_bin("target/release/my-app")?,
+///     std::collections::HashSet::from_iter([
+///         icu::list::provider::AndListV1Marker::INFO,
+///         icu::list::provider::OrListV1Marker::INFO,
+///     ]),
+/// );
+/// # Ok(())
+/// # }
+/// ```
+pub fn markers_for_bin<P: AsRef<Path>>(path: P) -> Result<HashSet<DataMarkerInfo>, DataError> {
+    let file = std::fs::read(path.as_ref())?;
+    let file = file.as_slice();
+
+    markers_for_bin_inner(file)
+}
+
+#[test]
+fn test_markers_for_bin() {
+    assert_eq!(
+        markers_for_bin_inner(include_bytes!("../tests/data/tutorial_buffer.wasm")),
+        Ok(HashSet::from_iter([
+            crate::datetime::provider::calendar::GregorianDateLengthsV1Marker::INFO,
+            crate::datetime::provider::calendar::GregorianDateSymbolsV1Marker::INFO,
+            crate::datetime::provider::calendar::TimeLengthsV1Marker::INFO,
+            crate::datetime::provider::calendar::TimeSymbolsV1Marker::INFO,
+            crate::calendar::provider::WeekDataV1Marker::INFO,
+            crate::decimal::provider::DecimalSymbolsV1Marker::INFO,
+            crate::plurals::provider::OrdinalV1Marker::INFO,
+        ]))
+    );
+}

components/icu/src/lib.rs

@@ -192,3 +192,8 @@ pub use icu_timezone as timezone;
 #[doc(inline)]
 #[cfg(feature = "experimental")]
 pub use icu_experimental as experimental;
+
+#[cfg(feature = "datagen")]
+mod datagen;
+#[cfg(feature = "datagen")]
+pub use datagen::*;

provider/baked/src/export.rs

@@ -21,11 +21,8 @@
 //! let mut exporter =
 //!     BakedExporter::new(demo_path.clone(), Default::default()).unwrap();
 //!
-//! // Export something
-//! DatagenDriver::new()
-//!     .with_markers([icu_provider::hello_world::HelloWorldV1Marker::INFO])
-//!     // HelloWorldProvider cannot provide fallback data, so we cannot deduplicate
-//!     .with_locales_and_fallback([LocaleFamily::FULL], FallbackOptions::no_deduplication())
+//! // Export something. Make sure to use the same fallback data at runtime!
+//! DatagenDriver::new([LocaleFamily::FULL], FallbackOptions::maximal_deduplication(), LocaleFallbacker::new().static_to_owned())
 //!     .export(&icu_provider::hello_world::HelloWorldProvider, exporter)
 //!     .unwrap();
 //! #

provider/bikeshed/Cargo.toml

@@ -73,7 +73,7 @@ num-traits = { workspace = true, optional = true }
 
 [dev-dependencies]
 postcard = { workspace = true, features = ["alloc"] }
-icu_datagen = { workspace = true, features = ["experimental", "fs_exporter", "baked_exporter", "rayon"] }
+icu_datagen = { workspace = true, features = ["fs_exporter", "baked_exporter", "rayon"] }
 icu_provider = { workspace = true, features = ["deserialize_postcard_1"] }
 icu_segmenter = { path = "../../components/segmenter", features = ["lstm"] }
 simple_logger = { workspace = true }

provider/bikeshed/src/tests/make_testdata.rs

@@ -53,18 +53,19 @@ fn make_testdata() {
         ]))
     };
 
-    DatagenDriver::new()
-        .with_markers(icu_datagen::all_markers())
-        .with_locales_and_fallback(
-            LOCALES.iter().cloned().map(LocaleFamily::with_descendants),
-            FallbackOptions::no_deduplication(),
-        )
-        .with_segmenter_models([
-            "thaidict".into(),
-            "Thai_codepoints_exclusive_model4_heavy".into(),
-        ])
-        .export(&DatagenProvider::new_testing(), exporter)
-        .unwrap()
+    let provider = DatagenProvider::new_testing();
+
+    DatagenDriver::new(
+        LOCALES.iter().cloned().map(LocaleFamily::with_descendants),
+        FallbackOptions::no_deduplication(),
+        LocaleFallbacker::try_new_unstable(&provider).unwrap(),
+    )
+    .with_segmenter_models([
+        "thaidict".into(),
+        "Thai_codepoints_exclusive_model4_heavy".into(),
+    ])
+    .export(&provider, exporter)
+    .unwrap()
 }
 
 struct ZeroCopyCheckExporter {

provider/blob/src/export/mod.rs

@@ -19,11 +19,8 @@
 //! // Set up the exporter
 //! let mut exporter = BlobExporter::new_v2_with_sink(Box::new(&mut blob));
 //!
-//! // Export something
-//! DatagenDriver::new()
-//!     .with_markers([icu_provider::hello_world::HelloWorldV1Marker::INFO])
-//!     // HelloWorldProvider cannot provide fallback data, so we cannot deduplicate
-//!     .with_locales_and_fallback([LocaleFamily::FULL], FallbackOptions::no_deduplication())
+//! // Export something. Make sure to use the same fallback data at runtime!
+//! DatagenDriver::new([LocaleFamily::FULL], FallbackOptions::maximal_deduplication(), LocaleFallbacker::new().static_to_owned())
 //!     .export(&icu_provider::hello_world::HelloWorldProvider, exporter)
 //!     .unwrap();
 //!

provider/core/src/datagen/mod.rs

@@ -20,6 +20,7 @@ pub use iter::IterableDynamicDataProvider;
 pub use payload::{ExportBox, ExportMarker};
 
 use crate::prelude::*;
+use std::collections::HashSet;
 
 /// An object capable of exporting data payloads in some form.
 pub trait DataExporter: Sync {
@@ -94,11 +95,14 @@ impl DataExporter for Box<dyn DataExporter> {
 pub trait ExportableProvider:
     IterableDynamicDataProvider<ExportMarker> + DynamicDataProvider<AnyMarker> + Sync
 {
+    /// Returns the set of supported markers
+    fn supported_markers(&self) -> HashSet<DataMarkerInfo>;
 }
 
-impl<T> ExportableProvider for T where
-    T: IterableDynamicDataProvider<ExportMarker> + DynamicDataProvider<AnyMarker> + Sync
-{
+impl ExportableProvider for Box<dyn ExportableProvider> {
+    fn supported_markers(&self) -> HashSet<DataMarkerInfo> {
+        (**self).supported_markers()
+    }
 }
 
 /// This macro can be used on a data provider to allow it to be used for data generation.
@@ -117,6 +121,18 @@ impl<T> ExportableProvider for T where
 #[doc(hidden)] // macro
 macro_rules! __make_exportable_provider {
     ($provider:ty, [ $($(#[$cfg:meta])? $struct_m:ty),+, ]) => {
+        impl $crate::datagen::ExportableProvider for $provider {
+            fn supported_markers(&self) -> std::collections::HashSet<$crate::DataMarkerInfo> {
+                std::collections::HashSet::from_iter([
+                    $(
+                        $(#[$cfg])?
+                        <$struct_m>::INFO,
+                    )+
+                ])
+            }
+        }
+
+
         $crate::dynutil::impl_dynamic_data_provider!(
             $provider,
             [ $($(#[$cfg])? $struct_m),+, ],

provider/datagen/Cargo.toml

@@ -33,13 +33,11 @@ all-features = true
 
 # DatagenDriver
 displaydoc = { workspace = true }
-icu = { workspace = true, features = ["std"] }
+icu_locale = { workspace = true }
 icu_provider = { workspace = true, features = ["std", "logging", "datagen"]}
 log = { workspace = true }
-memchr = { workspace = true }
 rayon = { workspace = true, optional = true }
 writeable = { workspace = true }
-icu_registry = { workspace = true }
 
 # Exporters
 icu_provider_blob = { workspace = true, features = ["export"], optional = true }
@@ -59,5 +57,3 @@ baked_exporter = ["dep:icu_provider_baked"]
 blob_exporter = ["dep:icu_provider_blob"]
 fs_exporter = ["dep:icu_provider_fs"]
 rayon = ["dep:rayon"]
-# For all_markers
-experimental = ["icu/experimental"]