feat: Validium with DA

.github/workflows/ci-core-reusable.yml

@@ -134,7 +134,7 @@ jobs:
         base_token: ["Eth", "Custom"]
         deployment_mode: ["Rollup", "Validium"]
     env:
-      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator${{ matrix.consensus && ',consensus' || '' }}"
+      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator${{ matrix.consensus && ',consensus' || '' }}${{ matrix.deployment_mode == 'Validium' && ',da_dispatcher' || '' }}"
 
     runs-on: [matterlabs-ci-runner]
     steps:
@@ -301,7 +301,7 @@ jobs:
     runs-on: [matterlabs-ci-runner]
 
     env:
-      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator${{ matrix.consensus && ',consensus' || '' }}"
+      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator${{ matrix.consensus && ',consensus' || '' }}${{ matrix.deployment_mode == 'Validium' && ',da_dispatcher' || '' }}"
       EXT_NODE_FLAGS: "${{ matrix.consensus && '-- --enable-consensus' || '' }}"
 
     steps:

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "core/node/shared_metrics",
     "core/node/db_pruner",
     "core/node/fee_model",
+    "core/node/da_dispatcher",
     "core/node/eth_sender",
     "core/node/vm_runner",
     "core/node/test_utils",
@@ -43,6 +44,7 @@ members = [
     "core/lib/circuit_breaker",
     "core/lib/dal",
     "core/lib/env_config",
+    "core/lib/da_client",
     "core/lib/eth_client",
     "core/lib/eth_signer",
     "core/lib/l1_contract_interface",
@@ -189,6 +191,7 @@ zk_evm_1_3_3 = { package = "zk_evm", git = "https://github.com/matter-labs/era-z
 zk_evm_1_4_0 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.4.0" }
 zk_evm_1_4_1 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.4.1" }
 zk_evm_1_5_0 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.5.0" }
+
 zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3e6f101ee4124308c4c974caaa259d524549b0c6" }
 zksync_consensus_bft = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3e6f101ee4124308c4c974caaa259d524549b0c6" }
 zksync_consensus_crypto = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3e6f101ee4124308c4c974caaa259d524549b0c6" }
@@ -199,6 +202,7 @@ zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter
 zksync_consensus_utils = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3e6f101ee4124308c4c974caaa259d524549b0c6" }
 zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3e6f101ee4124308c4c974caaa259d524549b0c6" }
 zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3e6f101ee4124308c4c974caaa259d524549b0c6" }
+zksync_da_layers = { package = "hyperchain_da", git = "https://github.com/matter-labs/hyperchain-da.git", rev = "5629d56ec3d851a7ff03e72c5e0765c55af595d5" }
 
 # "Local" dependencies
 multivm = { path = "core/lib/multivm" }
@@ -218,6 +222,7 @@ zksync_dal = { path = "core/lib/dal" }
 zksync_db_connection = { path = "core/lib/db_connection" }
 zksync_env_config = { path = "core/lib/env_config" }
 zksync_eth_client = { path = "core/lib/eth_client" }
+zksync_da_client = { path = "core/lib/da_client" }
 zksync_eth_signer = { path = "core/lib/eth_signer" }
 zksync_health_check = { path = "core/lib/health_check" }
 zksync_l1_contract_interface = { path = "core/lib/l1_contract_interface" }
@@ -248,6 +253,7 @@ zksync_block_reverter = { path = "core/node/block_reverter" }
 zksync_commitment_generator = { path = "core/node/commitment_generator" }
 zksync_house_keeper = { path = "core/node/house_keeper" }
 zksync_node_genesis = { path = "core/node/genesis" }
+zksync_da_dispatcher = { path = "core/node/da_dispatcher" }
 zksync_eth_sender = { path = "core/node/eth_sender" }
 zksync_node_db_pruner = { path = "core/node/db_pruner" }
 zksync_node_fee_model = { path = "core/node/fee_model" }

checks-config/era.dic

@@ -965,6 +965,7 @@ decrement
 Bbellman
 Sbellman
 DCMAKE
+blob_id
 preloaded
 e2e
 upcasting

core/bin/zksync_server/src/main.rs

@@ -15,8 +15,8 @@ use zksync_config::{
         FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig,
         L1Secrets, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, Secrets,
     },
-    ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig,
-    GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
+    ApiConfig, ContractVerifierConfig, DADispatcherConfig, DBConfig, EthConfig, EthWatchConfig,
+    GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
 };
 use zksync_core_leftovers::{
     genesis_init, initialize_components, is_genesis_needed, setup_sigint_handler,
@@ -48,7 +48,7 @@ struct Cli {
     /// Comma-separated list of components to launch.
     #[arg(
         long,
-        default_value = "api,tree,eth,state_keeper,housekeeper,tee_verifier_input_producer,commitment_generator"
+        default_value = "api,tree,eth,state_keeper,housekeeper,tee_verifier_input_producer,commitment_generator,da_dispatcher"
     )]
     components: ComponentsToRun,
     /// Path to the yaml config. If set, it will be used instead of env vars.
@@ -306,5 +306,6 @@ fn load_env_config() -> anyhow::Result<TempConfigStore> {
         object_store_config: ObjectStoreConfig::from_env().ok(),
         observability: ObservabilityConfig::from_env().ok(),
         snapshot_creator: SnapshotsCreatorConfig::from_env().ok(),
+        da_dispatcher_config: DADispatcherConfig::from_env().ok(),
     })
 }

core/bin/zksync_server/src/node_builder.rs

@@ -19,6 +19,7 @@ use zksync_node_framework::{
         commitment_generator::CommitmentGeneratorLayer,
         consensus::{ConsensusLayer, Mode as ConsensusMode},
         contract_verification_api::ContractVerificationApiLayer,
+        da_dispatcher::DataAvailabilityDispatcherLayer,
         eth_sender::{EthTxAggregatorLayer, EthTxManagerLayer},
         eth_watch::EthWatchLayer,
         healtcheck_server::HealthCheckLayer,
@@ -399,6 +400,16 @@ impl MainNodeBuilder {
         Ok(self)
     }
 
+    fn add_da_dispatcher_layer(mut self) -> anyhow::Result<Self> {
+        let eth_sender_config = try_load_config!(self.configs.eth);
+        let da_config = try_load_config!(self.configs.da_dispatcher_config);
+        self.node.add_layer(DataAvailabilityDispatcherLayer::new(
+            da_config,
+            eth_sender_config,
+        ));
+        Ok(self)
+    }
+
     pub fn build(mut self, mut components: Vec<Component>) -> anyhow::Result<ZkStackService> {
         // Add "base" layers (resources and helper tasks).
         self = self
@@ -480,6 +491,9 @@ impl MainNodeBuilder {
                 Component::CommitmentGenerator => {
                     self = self.add_commitment_generator_layer()?;
                 }
+                Component::DADispatcher => {
+                    self = self.add_da_dispatcher_layer()?;
+                }
             }
         }
         Ok(self.node.build()?)

core/lib/config/src/configs/da_dispatcher.rs

@@ -0,0 +1,58 @@
+use std::time::Duration;
+
+use serde::Deserialize;
+
+use crate::ObjectStoreConfig;
+
+#[derive(Clone, Debug, PartialEq, Deserialize)]
+pub struct DALayerInfo {
+    pub name: String,
+    #[serde(default)]
+    pub private_key: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Deserialize)]
+#[serde(tag = "da_mode")]
+pub enum DataAvailabilityMode {
+    DALayer(DALayerInfo),
+    GCS(ObjectStoreConfig),
+    NoDA,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct DADispatcherConfig {
+    #[serde(flatten)]
+    pub da_mode: DataAvailabilityMode,
+    pub polling_interval: Option<u32>,
+    pub query_rows_limit: Option<u32>,
+    pub max_retries: Option<u16>,
+}
+
+impl DADispatcherConfig {
+    pub fn for_tests() -> Self {
+        Self {
+            da_mode: DataAvailabilityMode::DALayer(DALayerInfo {
+                name: "zkDA".into(),
+                private_key: "0x0".into(),
+            }),
+            polling_interval: Some(5),
+            query_rows_limit: Some(100),
+            max_retries: Some(5),
+        }
+    }
+
+    pub fn polling_interval(&self) -> Duration {
+        match self.polling_interval {
+            Some(interval) => Duration::from_secs(interval as u64),
+            None => Duration::from_secs(5),
+        }
+    }
+
+    pub fn query_rows_limit(&self) -> u32 {
+        self.query_rows_limit.unwrap_or(100)
+    }
+
+    pub fn max_retries(&self) -> u16 {
+        self.max_retries.unwrap_or(5)
+    }
+}

core/lib/config/src/configs/eth_sender.rs

@@ -81,6 +81,7 @@ pub enum PubdataSendingMode {
     #[default]
     Calldata,
     Blobs,
+    Custom,
 }
 
 #[derive(Debug, Deserialize, Clone, PartialEq)]
@@ -114,7 +115,7 @@ pub struct SenderConfig {
     // Max acceptable fee for sending tx it acts as a safeguard to prevent sending tx with very high fees.
     pub max_acceptable_priority_fee_in_gwei: u64,
 
-    /// The mode in which we send pubdata, either Calldata or Blobs
+    /// The mode in which we send pubdata
     pub pubdata_sending_mode: PubdataSendingMode,
 }
 

core/lib/config/src/configs/general.rs

@@ -1,6 +1,7 @@
 use crate::{
     configs::{
         chain::{CircuitBreakerConfig, MempoolConfig, OperationsManagerConfig, StateKeeperConfig},
+        da_dispatcher::DADispatcherConfig,
         fri_prover_group::FriProverGroupConfig,
         house_keeper::HouseKeeperConfig,
         FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig,
@@ -32,4 +33,5 @@ pub struct GeneralConfig {
     pub eth: Option<EthConfig>,
     pub snapshot_creator: Option<SnapshotsCreatorConfig>,
     pub observability: Option<ObservabilityConfig>,
+    pub da_dispatcher_config: Option<DADispatcherConfig>,
 }

core/lib/config/src/configs/mod.rs

@@ -3,6 +3,7 @@ pub use self::{
     api::ApiConfig,
     contract_verifier::ContractVerifierConfig,
     contracts::{ContractsConfig, EcosystemContracts},
+    da_dispatcher::DADispatcherConfig,
     database::{DBConfig, PostgresConfig},
     eth_sender::{EthConfig, GasAdjusterConfig},
     eth_watch::EthWatchConfig,
@@ -27,6 +28,7 @@ pub mod chain;
 pub mod consensus;
 pub mod contract_verifier;
 pub mod contracts;
+pub mod da_dispatcher;
 pub mod database;
 pub mod eth_sender;
 pub mod eth_watch;

core/lib/config/src/lib.rs

@@ -1,8 +1,9 @@
 #![allow(clippy::upper_case_acronyms, clippy::derive_partial_eq_without_eq)]
 
 pub use crate::configs::{
-    ApiConfig, ContractVerifierConfig, ContractsConfig, DBConfig, EthConfig, EthWatchConfig,
-    GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
+    ApiConfig, ContractVerifierConfig, ContractsConfig, DADispatcherConfig, DBConfig, EthConfig,
+    EthWatchConfig, GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig,
+    SnapshotsCreatorConfig,
 };
 
 pub mod configs;

core/lib/config/src/testonly.rs

@@ -340,9 +340,10 @@ impl Distribution<configs::eth_sender::ProofLoadingMode> for EncodeDist {
 impl Distribution<configs::eth_sender::PubdataSendingMode> for EncodeDist {
     fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::eth_sender::PubdataSendingMode {
         type T = configs::eth_sender::PubdataSendingMode;
-        match rng.gen_range(0..2) {
+        match rng.gen_range(0..3) {
             0 => T::Calldata,
-            _ => T::Blobs,
+            1 => T::Blobs,
+            _ => T::Custom,
         }
     }
 }

core/lib/da_client/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "zksync_da_client"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1.74"
+
+zksync_config.workspace = true
+zksync_types.workspace = true
+zksync_da_layers.workspace = true
+zksync_object_store.workspace = true
+tracing = "0.1.40"