From 298a97e800b4c156628050789de7a490a7565d60 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Wed, 26 Jun 2024 09:35:54 +0400 Subject: [PATCH 1/6] feat(node_framework): Unify Task types + misc improvements (#2325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ - Unifies `Task` types. Now we only have a single `Task` type with different `TaskKind` specifiers. - Refactors `ZkStackService::run` so that it's more readable. - Updates the framework documentation. - Minor improvements here and there. ## Why ❔ - Preparing framework for the previously proposed refactoring (e.g. `FromContext` / `IntoContext` IO flow). - Preparing framework for the publishing. ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. --- core/node/node_framework/examples/showcase.rs | 2 - .../layers/circuit_breaker_checker.rs | 15 +- .../layers/healtcheck_server.rs | 15 +- .../l1_batch_commitment_mode_validation.rs | 13 +- .../layers/postgres_metrics.rs | 15 +- .../layers/prometheus_exporter.rs | 12 +- .../layers/reorg_detector_checker.rs | 13 +- .../layers/reorg_detector_runner.rs | 15 +- .../src/implementations/layers/sigint.rs | 15 +- .../layers/validate_chain_ids.rs | 13 +- .../implementations/resources/sync_state.rs | 2 +- core/node/node_framework/src/lib.rs | 11 +- core/node/node_framework/src/precondition.rs | 41 ---- .../src/resource/lazy_resource.rs | 194 --------------- core/node/node_framework/src/resource/mod.rs | 41 +++- .../src/resource/resource_collection.rs | 172 -------------- .../node_framework/src/resource/unique.rs | 1 + .../node_framework/src/service/context.rs | 91 +++----- core/node/node_framework/src/service/error.rs | 2 + core/node/node_framework/src/service/mod.rs | 206 ++++++++-------- .../src/service/named_future.rs | 23 +- .../node_framework/src/service/runnables.rs | 204 +++++++--------- .../src/service/stop_receiver.rs | 6 - core/node/node_framework/src/task.rs | 221 ------------------ core/node/node_framework/src/task/mod.rs | 138 +++++++++++ core/node/node_framework/src/task/types.rs | 60 +++++ 26 files changed, 530 insertions(+), 1011 deletions(-) delete mode 100644 core/node/node_framework/src/precondition.rs delete mode 100644 core/node/node_framework/src/resource/lazy_resource.rs delete mode 100644 core/node/node_framework/src/resource/resource_collection.rs delete mode 100644 core/node/node_framework/src/task.rs create mode 100644 core/node/node_framework/src/task/mod.rs create mode 100644 core/node/node_framework/src/task/types.rs diff --git a/core/node/node_framework/examples/showcase.rs b/core/node/node_framework/examples/showcase.rs index 98baa5bc9683..67fa819880bc 100644 --- a/core/node/node_framework/examples/showcase.rs +++ b/core/node/node_framework/examples/showcase.rs @@ -63,8 +63,6 @@ struct DatabaseResource(pub Arc); /// /// For the latter requirement, there exists an `Unique` wrapper that can be used to store non-`Clone` /// resources. It's not used in this example, but it's a useful thing to know about. -/// -/// Finally, there are other wrappers for resources as well, like `ResourceCollection` and `LazyResource`. impl Resource for DatabaseResource { fn name() -> String { // The convention for resource names is `/`. In this case, the scope is `common`, but diff --git a/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs b/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs index 808ac7f57774..d7334147bdc2 100644 --- a/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs +++ b/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs @@ -4,7 +4,7 @@ use zksync_config::configs::chain::CircuitBreakerConfig; use crate::{ implementations::resources::circuit_breakers::CircuitBreakersResource, service::{ServiceContext, StopReceiver}, - task::{TaskId, UnconstrainedTask}, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -44,7 +44,7 @@ impl WiringLayer for CircuitBreakerCheckerLayer { circuit_breaker_checker, }; - node.add_unconstrained_task(Box::new(task)); + node.add_task(Box::new(task)); Ok(()) } } @@ -55,15 +55,16 @@ struct CircuitBreakerCheckerTask { } #[async_trait::async_trait] -impl UnconstrainedTask for CircuitBreakerCheckerTask { +impl Task for CircuitBreakerCheckerTask { + fn kind(&self) -> TaskKind { + TaskKind::UnconstrainedTask + } + fn id(&self) -> TaskId { "circuit_breaker_checker".into() } - async fn run_unconstrained( - mut self: Box, - stop_receiver: StopReceiver, - ) -> anyhow::Result<()> { + async fn run(mut self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { self.circuit_breaker_checker.run(stop_receiver.0).await } } diff --git a/core/node/node_framework/src/implementations/layers/healtcheck_server.rs b/core/node/node_framework/src/implementations/layers/healtcheck_server.rs index 10f98d8f9e5a..3982044c3f96 100644 --- a/core/node/node_framework/src/implementations/layers/healtcheck_server.rs +++ b/core/node/node_framework/src/implementations/layers/healtcheck_server.rs @@ -7,7 +7,7 @@ use zksync_node_api_server::healthcheck::HealthCheckHandle; use crate::{ implementations::resources::healthcheck::AppHealthCheckResource, service::{ServiceContext, StopReceiver}, - task::{TaskId, UnconstrainedTask}, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -41,7 +41,7 @@ impl WiringLayer for HealthCheckLayer { app_health_check, }; - node.add_unconstrained_task(Box::new(task)); + node.add_task(Box::new(task)); Ok(()) } } @@ -53,15 +53,16 @@ struct HealthCheckTask { } #[async_trait::async_trait] -impl UnconstrainedTask for HealthCheckTask { +impl Task for HealthCheckTask { + fn kind(&self) -> TaskKind { + TaskKind::UnconstrainedTask + } + fn id(&self) -> TaskId { "healthcheck_server".into() } - async fn run_unconstrained( - mut self: Box, - mut stop_receiver: StopReceiver, - ) -> anyhow::Result<()> { + async fn run(mut self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { let handle = HealthCheckHandle::spawn_server(self.config.bind_addr(), self.app_health_check.clone()); stop_receiver.0.changed().await?; diff --git a/core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs b/core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs index b9a83cc06cb6..3bb82dde98b6 100644 --- a/core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs +++ b/core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs @@ -3,9 +3,8 @@ use zksync_types::{commitment::L1BatchCommitmentMode, Address}; use crate::{ implementations::resources::eth_interface::EthInterfaceResource, - precondition::Precondition, service::{ServiceContext, StopReceiver}, - task::TaskId, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -51,19 +50,23 @@ impl WiringLayer for L1BatchCommitmentModeValidationLayer { query_client, ); - context.add_precondition(Box::new(task)); + context.add_task(Box::new(task)); Ok(()) } } #[async_trait::async_trait] -impl Precondition for L1BatchCommitmentModeValidationTask { +impl Task for L1BatchCommitmentModeValidationTask { + fn kind(&self) -> TaskKind { + TaskKind::Precondition + } + fn id(&self) -> TaskId { "l1_batch_commitment_mode_validation".into() } - async fn check(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { (*self).exit_on_success().run(stop_receiver.0).await } } diff --git a/core/node/node_framework/src/implementations/layers/postgres_metrics.rs b/core/node/node_framework/src/implementations/layers/postgres_metrics.rs index a0c80d4e9d42..b0690880a4c1 100644 --- a/core/node/node_framework/src/implementations/layers/postgres_metrics.rs +++ b/core/node/node_framework/src/implementations/layers/postgres_metrics.rs @@ -5,7 +5,7 @@ use zksync_dal::{metrics::PostgresMetrics, ConnectionPool, Core}; use crate::{ implementations::resources::pools::{PoolResource, ReplicaPool}, service::{ServiceContext, StopReceiver}, - task::{TaskId, UnconstrainedTask}, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -32,7 +32,7 @@ impl WiringLayer for PostgresMetricsLayer { async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { let replica_pool_resource = context.get_resource::>().await?; let pool_for_metrics = replica_pool_resource.get_singleton().await?; - context.add_unconstrained_task(Box::new(PostgresMetricsScrapingTask { pool_for_metrics })); + context.add_task(Box::new(PostgresMetricsScrapingTask { pool_for_metrics })); Ok(()) } @@ -44,15 +44,16 @@ struct PostgresMetricsScrapingTask { } #[async_trait::async_trait] -impl UnconstrainedTask for PostgresMetricsScrapingTask { +impl Task for PostgresMetricsScrapingTask { + fn kind(&self) -> TaskKind { + TaskKind::UnconstrainedTask + } + fn id(&self) -> TaskId { "postgres_metrics_scraping".into() } - async fn run_unconstrained( - self: Box, - mut stop_receiver: StopReceiver, - ) -> anyhow::Result<()> { + async fn run(self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { tokio::select! { () = PostgresMetrics::run_scraping(self.pool_for_metrics, SCRAPE_INTERVAL) => { tracing::warn!("Postgres metrics scraping unexpectedly stopped"); diff --git a/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs b/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs index 0742de55e2db..91b205f38cd8 100644 --- a/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs +++ b/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs @@ -4,7 +4,7 @@ use zksync_vlog::prometheus::PrometheusExporterConfig; use crate::{ implementations::resources::healthcheck::AppHealthCheckResource, service::{ServiceContext, StopReceiver}, - task::{TaskId, UnconstrainedTask}, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -46,18 +46,22 @@ impl WiringLayer for PrometheusExporterLayer { prometheus_health_updater, }); - node.add_unconstrained_task(task); + node.add_task(task); Ok(()) } } #[async_trait::async_trait] -impl UnconstrainedTask for PrometheusExporterTask { +impl Task for PrometheusExporterTask { + fn kind(&self) -> TaskKind { + TaskKind::UnconstrainedTask + } + fn id(&self) -> TaskId { "prometheus_exporter".into() } - async fn run_unconstrained(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { let prometheus_task = self.config.run(stop_receiver.0); self.prometheus_health_updater .update(HealthStatus::Ready.into()); diff --git a/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs b/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs index 31b93a1b566e..a55c8a5e74ab 100644 --- a/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs +++ b/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs @@ -9,9 +9,8 @@ use crate::{ main_node_client::MainNodeClientResource, pools::{MasterPool, PoolResource}, }, - precondition::Precondition, service::{ServiceContext, StopReceiver}, - task::TaskId, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -45,7 +44,7 @@ impl WiringLayer for ReorgDetectorCheckerLayer { let pool = pool_resource.get().await?; // Create and insert precondition. - context.add_precondition(Box::new(CheckerPrecondition { + context.add_task(Box::new(CheckerPrecondition { pool: pool.clone(), reorg_detector: ReorgDetector::new(main_node_client, pool), })); @@ -60,12 +59,16 @@ pub struct CheckerPrecondition { } #[async_trait::async_trait] -impl Precondition for CheckerPrecondition { +impl Task for CheckerPrecondition { + fn kind(&self) -> TaskKind { + TaskKind::Precondition + } + fn id(&self) -> TaskId { "reorg_detector_checker".into() } - async fn check(mut self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { + async fn run(mut self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { // Given that this is a precondition -- i.e. something that starts before some invariants are met, // we need to first ensure that there is at least one batch in the database (there may be none if // either genesis or snapshot recovery has not been performed yet). diff --git a/core/node/node_framework/src/implementations/layers/reorg_detector_runner.rs b/core/node/node_framework/src/implementations/layers/reorg_detector_runner.rs index 2ffc33d3145b..ab0995f10211 100644 --- a/core/node/node_framework/src/implementations/layers/reorg_detector_runner.rs +++ b/core/node/node_framework/src/implementations/layers/reorg_detector_runner.rs @@ -11,7 +11,7 @@ use crate::{ reverter::BlockReverterResource, }, service::{ServiceContext, StopReceiver}, - task::{TaskId, UnconstrainedOneshotTask}, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -46,7 +46,7 @@ impl WiringLayer for ReorgDetectorRunnerLayer { let reverter = context.get_resource::().await?.0; // Create and insert task. - context.add_unconstrained_oneshot_task(Box::new(RunnerUnconstrainedOneshotTask { + context.add_task(Box::new(RunnerUnconstrainedOneshotTask { reorg_detector: ReorgDetector::new(main_node_client, pool), reverter, })); @@ -61,15 +61,16 @@ pub struct RunnerUnconstrainedOneshotTask { } #[async_trait::async_trait] -impl UnconstrainedOneshotTask for RunnerUnconstrainedOneshotTask { +impl Task for RunnerUnconstrainedOneshotTask { + fn kind(&self) -> TaskKind { + TaskKind::UnconstrainedOneshotTask + } + fn id(&self) -> TaskId { "reorg_detector_runner".into() } - async fn run_unconstrained_oneshot( - mut self: Box, - stop_receiver: StopReceiver, - ) -> anyhow::Result<()> { + async fn run(mut self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { match self.reorg_detector.run_once(stop_receiver.0.clone()).await { Ok(()) => {} Err(zksync_reorg_detector::Error::ReorgDetected(last_correct_l1_batch)) => { diff --git a/core/node/node_framework/src/implementations/layers/sigint.rs b/core/node/node_framework/src/implementations/layers/sigint.rs index c3200139aba9..5c1fab73fa16 100644 --- a/core/node/node_framework/src/implementations/layers/sigint.rs +++ b/core/node/node_framework/src/implementations/layers/sigint.rs @@ -2,7 +2,7 @@ use tokio::sync::oneshot; use crate::{ service::{ServiceContext, StopReceiver}, - task::{TaskId, UnconstrainedTask}, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -23,7 +23,7 @@ impl WiringLayer for SigintHandlerLayer { async fn wire(self: Box, mut node: ServiceContext<'_>) -> Result<(), WiringError> { // SIGINT may happen at any time, so we must handle it as soon as it happens. - node.add_unconstrained_task(Box::new(SigintHandlerTask)); + node.add_task(Box::new(SigintHandlerTask)); Ok(()) } } @@ -32,15 +32,16 @@ impl WiringLayer for SigintHandlerLayer { struct SigintHandlerTask; #[async_trait::async_trait] -impl UnconstrainedTask for SigintHandlerTask { +impl Task for SigintHandlerTask { + fn kind(&self) -> TaskKind { + TaskKind::UnconstrainedTask + } + fn id(&self) -> TaskId { "sigint_handler".into() } - async fn run_unconstrained( - self: Box, - mut stop_receiver: StopReceiver, - ) -> anyhow::Result<()> { + async fn run(self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { let (sigint_sender, sigint_receiver) = oneshot::channel(); let mut sigint_sender = Some(sigint_sender); // Has to be done this way since `set_handler` requires `FnMut`. ctrlc::set_handler(move || { diff --git a/core/node/node_framework/src/implementations/layers/validate_chain_ids.rs b/core/node/node_framework/src/implementations/layers/validate_chain_ids.rs index a9f5a61c65f1..5d3a9b9e82f5 100644 --- a/core/node/node_framework/src/implementations/layers/validate_chain_ids.rs +++ b/core/node/node_framework/src/implementations/layers/validate_chain_ids.rs @@ -5,9 +5,8 @@ use crate::{ implementations::resources::{ eth_interface::EthInterfaceResource, main_node_client::MainNodeClientResource, }, - precondition::Precondition, service::{ServiceContext, StopReceiver}, - task::TaskId, + task::{Task, TaskId, TaskKind}, wiring_layer::{WiringError, WiringLayer}, }; @@ -54,19 +53,23 @@ impl WiringLayer for ValidateChainIdsLayer { main_node_client, ); - context.add_precondition(Box::new(task)); + context.add_task(Box::new(task)); Ok(()) } } #[async_trait::async_trait] -impl Precondition for ValidateChainIdsTask { +impl Task for ValidateChainIdsTask { + fn kind(&self) -> TaskKind { + TaskKind::Precondition + } + fn id(&self) -> TaskId { "validate_chain_ids".into() } - async fn check(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { (*self).run_once(stop_receiver.0).await } } diff --git a/core/node/node_framework/src/implementations/resources/sync_state.rs b/core/node/node_framework/src/implementations/resources/sync_state.rs index 25df1d94d99f..a65342dd38d6 100644 --- a/core/node/node_framework/src/implementations/resources/sync_state.rs +++ b/core/node/node_framework/src/implementations/resources/sync_state.rs @@ -8,6 +8,6 @@ pub struct SyncStateResource(pub SyncState); impl Resource for SyncStateResource { fn name() -> String { - "sync_state".into() + "common/sync_state".into() } } diff --git a/core/node/node_framework/src/lib.rs b/core/node/node_framework/src/lib.rs index 4f688ab56adb..da788609b57c 100644 --- a/core/node/node_framework/src/lib.rs +++ b/core/node/node_framework/src/lib.rs @@ -1,25 +1,16 @@ //! # ZK Stack node initialization framework. //! -//! ## Introduction -//! //! This crate provides core abstractions that allow one to compose a ZK Stack node. //! Main concepts used in this crate are: //! - [`WiringLayer`](wiring_layer::WiringLayer) - builder interface for tasks. //! - [`Task`](task::Task) - a unit of work that can be executed by the node. //! - [`Resource`](resource::Resource) - a piece of logic that can be shared between tasks. Most resources are //! represented by generic interfaces and also serve as points of customization for tasks. -//! - [`ResourceProvider`](resource::ResourceProvider) - a trait that allows one to provide resources to the node. //! - [`ZkStackService`](service::ZkStackService) - a container for tasks and resources that takes care of initialization, running //! and shutting down. -//! -//! The general flow to compose a node is as follows: -//! - Create a [`ResourceProvider`](resource::ResourceProvider) that can provide all the resources that the node needs. -//! - Create a [`ZkStackService`](node::ZkStackService) with that [`ResourceProvider`](resource::ResourceProvider). -//! - Add tasks to the node. -//! - Run it. +//! - [`ZkStackServiceBuilder`](service::ZkStackServiceBuilder) - a builder for the service. pub mod implementations; -pub mod precondition; pub mod resource; pub mod service; pub mod task; diff --git a/core/node/node_framework/src/precondition.rs b/core/node/node_framework/src/precondition.rs deleted file mode 100644 index d81e0328bb62..000000000000 --- a/core/node/node_framework/src/precondition.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::{fmt, sync::Arc}; - -use tokio::sync::Barrier; - -use crate::{service::StopReceiver, task::TaskId}; - -#[async_trait::async_trait] -pub trait Precondition: 'static + Send + Sync { - /// Unique name of the precondition. - fn id(&self) -> TaskId; - - async fn check(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()>; -} - -impl dyn Precondition { - /// An internal helper method that runs a precondition check and lifts the barrier as soon - /// as the check is finished. - pub(super) async fn check_with_barrier( - self: Box, - mut stop_receiver: StopReceiver, - preconditions_barrier: Arc, - ) -> anyhow::Result<()> { - self.check(stop_receiver.clone()).await?; - tokio::select! { - _ = preconditions_barrier.wait() => { - Ok(()) - } - _ = stop_receiver.0.changed() => { - Ok(()) - } - } - } -} - -impl fmt::Debug for dyn Precondition { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Precondition") - .field("name", &self.id()) - .finish() - } -} diff --git a/core/node/node_framework/src/resource/lazy_resource.rs b/core/node/node_framework/src/resource/lazy_resource.rs deleted file mode 100644 index 3f70187627b8..000000000000 --- a/core/node/node_framework/src/resource/lazy_resource.rs +++ /dev/null @@ -1,194 +0,0 @@ -use std::sync::Arc; - -use thiserror::Error; -use tokio::sync::watch; - -use super::Resource; -use crate::service::StopReceiver; - -/// A lazy resource represents a resource that isn't available at the time when the tasks start. -/// -/// Normally it's used to represent the resources that should be provided by one task to another one. -/// Lazy resources are aware of the node lifecycle, so attempt to resolve the resource won't hang -/// if the resource is never provided: the resolve future will fail once the stop signal is sent by the node. -#[derive(Debug)] -pub struct LazyResource { - resolve_sender: Arc>>, - stop_receiver: StopReceiver, -} - -impl Resource for LazyResource { - fn name() -> String { - format!("lazy {}", T::name()) - } -} - -impl Clone for LazyResource { - fn clone(&self) -> Self { - Self { - resolve_sender: self.resolve_sender.clone(), - stop_receiver: self.stop_receiver.clone(), - } - } -} - -impl LazyResource { - /// Creates a new lazy resource. - /// Provided stop receiver will be used to prevent resolving from hanging if the resource is never provided. - pub fn new(stop_receiver: StopReceiver) -> Self { - let (resolve_sender, _resolve_receiver) = watch::channel(None); - - Self { - resolve_sender: Arc::new(resolve_sender), - stop_receiver, - } - } - - /// Returns a future that resolves to the resource once it is provided. - /// If the resource is never provided, the method will return an error once the node is shutting down. - pub async fn resolve(mut self) -> Result { - let mut resolve_receiver = self.resolve_sender.subscribe(); - if let Some(resource) = resolve_receiver.borrow().as_ref() { - return Ok(resource.clone()); - } - - let result = tokio::select! { - _ = self.stop_receiver.0.changed() => { - Err(LazyResourceError::NodeShutdown) - } - _ = resolve_receiver.changed() => { - // ^ we can ignore the error on `changed`, since we hold a strong reference to the sender. - let resource = resolve_receiver.borrow().as_ref().expect("Can only change if provided").clone(); - Ok(resource) - } - }; - - if result.is_ok() { - tracing::info!("Lazy resource {} has been resolved", T::name()); - } - - result - } - - /// Provides the resource. - /// May be called at most once. Subsequent calls will return an error. - pub async fn provide(&mut self, resource: T) -> Result<(), LazyResourceError> { - let sent = self.resolve_sender.send_if_modified(|current| { - if current.is_some() { - return false; - } - *current = Some(resource.clone()); - true - }); - - if !sent { - return Err(LazyResourceError::ResourceAlreadyProvided); - } - - tracing::info!("Lazy resource {} has been provided", T::name()); - - Ok(()) - } -} - -#[derive(Debug, Error, PartialEq)] -pub enum LazyResourceError { - #[error("Node is shutting down")] - NodeShutdown, - #[error("Resource is already provided")] - ResourceAlreadyProvided, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[derive(Debug, Clone, PartialEq)] - struct TestResource(Arc); - - impl Resource for TestResource { - fn name() -> String { - "test_resource".into() - } - } - - struct TestContext { - test_resource: TestResource, - lazy_resource: LazyResource, - stop_sender: watch::Sender, - } - - impl TestContext { - fn new() -> Self { - let (stop_sender, stop_receiver) = watch::channel(false); - Self { - test_resource: TestResource(Arc::new(1)), - lazy_resource: LazyResource::::new(StopReceiver(stop_receiver)), - stop_sender, - } - } - } - - #[tokio::test] - async fn test_already_provided_resource_case() { - let TestContext { - test_resource, - lazy_resource, - stop_sender: _, - } = TestContext::new(); - - lazy_resource - .clone() - .provide(test_resource.clone()) - .await - .unwrap(); - - assert_eq!( - lazy_resource.clone().provide(test_resource.clone()).await, - Err(LazyResourceError::ResourceAlreadyProvided), - "Incorrect result for providing same resource twice" - ); - } - - #[tokio::test] - async fn test_successful_resolve_case() { - let TestContext { - test_resource, - lazy_resource, - stop_sender: _, - } = TestContext::new(); - - lazy_resource - .clone() - .provide(test_resource.clone()) - .await - .unwrap(); - - assert_eq!( - lazy_resource.clone().resolve().await, - Ok(test_resource.clone()), - "Incorrect result for resolving the resource before node shutdown" - ); - } - - #[tokio::test] - async fn test_node_shutdown_case() { - let TestContext { - test_resource: _, - lazy_resource, - stop_sender, - } = TestContext::new(); - - let resolve_task = tokio::spawn(async move { lazy_resource.resolve().await }); - - stop_sender.send(true).unwrap(); - - let result = resolve_task.await.unwrap(); - - assert_eq!( - result, - Err(LazyResourceError::NodeShutdown), - "Incorrect result for resolving the resource after the node shutdown" - ); - } -} diff --git a/core/node/node_framework/src/resource/mod.rs b/core/node/node_framework/src/resource/mod.rs index cf000acf8bb0..2e62d8421f89 100644 --- a/core/node/node_framework/src/resource/mod.rs +++ b/core/node/node_framework/src/resource/mod.rs @@ -1,12 +1,7 @@ use std::{any::TypeId, fmt}; -pub use self::{ - lazy_resource::LazyResource, resource_collection::ResourceCollection, resource_id::ResourceId, - unique::Unique, -}; +pub use self::{resource_id::ResourceId, unique::Unique}; -mod lazy_resource; -mod resource_collection; mod resource_id; mod unique; @@ -14,9 +9,39 @@ mod unique; /// Typically, the type that implements this trait also should implement `Clone` /// since the same resource may be requested by several tasks and thus it would be an additional /// bound on most methods that work with [`Resource`]. +/// +/// # Example +/// +/// ``` +/// # use zksync_node_framework::resource::Resource; +/// # use std::sync::Arc; +/// +/// /// An abstract interface you want to share. +/// /// Normally you want the interface to be thread-safe. +/// trait MyInterface: 'static + Send + Sync { +/// fn do_something(&self); +/// } +/// +/// /// Resource wrapper. +/// #[derive(Clone)] +/// struct MyResource(Arc); +/// +/// impl Resource for MyResource { +/// fn name() -> String { +/// // It is a helpful practice to follow a structured naming pattern for resource names. +/// // For example, you can use a certain prefix for all resources related to a some component, e.g. `api`. +/// "common/my_resource".to_string() +/// } +/// } +/// ``` pub trait Resource: 'static + Send + Sync + std::any::Any { + /// Invoked after the wiring phase of the service is done. + /// Can be used to perform additional resource preparation, knowing that the resource + /// is guaranteed to be requested by all the tasks that need it. fn on_resource_wired(&mut self) {} + /// Returns the name of the resource. + /// Used for logging purposes. fn name() -> String; } @@ -26,10 +51,10 @@ pub trait Resource: 'static + Send + Sync + std::any::Any { /// This trait is implemented for any type that implements [`Resource`], so there is no need to /// implement it manually. pub(crate) trait StoredResource: 'static + std::any::Any + Send + Sync { - /// An object-safe version of [`Resource::resource_id`]. + /// An object-safe version of [`Resource::name`]. fn stored_resource_id(&self) -> ResourceId; - /// An object-safe version of [`Resource::on_resoure_wired`]. + /// An object-safe version of [`Resource::on_resource_wired`]. fn stored_resource_wired(&mut self); } diff --git a/core/node/node_framework/src/resource/resource_collection.rs b/core/node/node_framework/src/resource/resource_collection.rs deleted file mode 100644 index 7f867f236d95..000000000000 --- a/core/node/node_framework/src/resource/resource_collection.rs +++ /dev/null @@ -1,172 +0,0 @@ -use std::{ - fmt, - sync::{Arc, Mutex}, -}; - -use thiserror::Error; -use tokio::sync::watch; - -use super::Resource; - -/// Collection of resources that can be extended during the initialization phase, and then resolved once -/// the wiring is complete. -/// -/// During component initialization, resource collections can be requested by the components in order to push new -/// elements there. Once the initialization is complete, it is no longer possible to push new elements, and the -/// collection can be resolved into a vector of resources. -/// -/// Collections implement `Clone`, so they can be consumed by several tasks. Every task that resolves the collection -/// is guaranteed to have the same set of resources. -/// -/// The purpose of this container is to allow different tasks to register their resource in a single place for some -/// other task to consume. For example, tasks may register their healthchecks, and then healthcheck task will observe -/// all the provided healthchecks. -pub struct ResourceCollection { - /// Collection of the resources. - resources: Arc>>, - /// Sender indicating that the wiring is complete. - wiring_complete_sender: Arc>, - /// Flag indicating that the collection has been resolved. - wired: watch::Receiver, -} - -impl Resource for ResourceCollection { - fn on_resource_wired(&mut self) { - self.wiring_complete_sender.send(true).ok(); - } - - fn name() -> String { - format!("collection of {}", T::name()) - } -} - -impl Default for ResourceCollection { - fn default() -> Self { - Self::new() - } -} - -impl Clone for ResourceCollection { - fn clone(&self) -> Self { - Self { - resources: self.resources.clone(), - wiring_complete_sender: self.wiring_complete_sender.clone(), - wired: self.wired.clone(), - } - } -} - -impl fmt::Debug for ResourceCollection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ResourceCollection") - .field("resources", &"{..}") - .finish_non_exhaustive() - } -} - -#[derive(Debug, Error)] -pub enum ResourceCollectionError { - #[error("Adding resources to the collection is not allowed after the wiring is complete")] - AlreadyWired, -} - -impl ResourceCollection { - pub(crate) fn new() -> Self { - let (wiring_complete_sender, wired) = watch::channel(false); - Self { - resources: Arc::default(), - wiring_complete_sender: Arc::new(wiring_complete_sender), - wired, - } - } - - /// Adds a new element to the resource collection. - /// Returns an error if the wiring is already complete. - pub fn push(&self, resource: T) -> Result<(), ResourceCollectionError> { - // This check is sufficient, since no task is guaranteed to be running when the value changes. - if *self.wired.borrow() { - return Err(ResourceCollectionError::AlreadyWired); - } - - let mut handle = self.resources.lock().unwrap(); - handle.push(resource); - tracing::info!( - "A new item has been added to the resource collection {}", - Self::name() - ); - Ok(()) - } - - /// Waits until the wiring is complete, and resolves the collection into a vector of resources. - pub async fn resolve(mut self) -> Vec { - // Guaranteed not to hang on server shutdown, since the node will invoke the `on_wiring_complete` before any task - // is actually spawned (per framework rules). For most cases, this check will resolve immediately, unless - // some tasks would spawn something from the `IntoZkSyncTask` impl. - self.wired.changed().await.expect("Sender can't be dropped"); - - tracing::info!("Resource collection {} has been resolved", Self::name()); - - let handle = self.resources.lock().unwrap(); - (*handle).clone() - } -} - -#[cfg(test)] -mod tests { - use assert_matches::assert_matches; - use futures::FutureExt; - - use super::*; - - #[derive(Debug, Clone, PartialEq)] - struct TestResource(Arc); - - impl Resource for TestResource { - fn name() -> String { - "test_resource".into() - } - } - - #[test] - fn test_push() { - let collection = ResourceCollection::::new(); - let resource1 = TestResource(Arc::new(1)); - collection.clone().push(resource1.clone()).unwrap(); - - let resource2 = TestResource(Arc::new(2)); - collection.clone().push(resource2.clone()).unwrap(); - - assert_eq!( - *collection.resources.lock().unwrap(), - vec![resource1, resource2] - ); - } - - #[test] - fn test_already_wired() { - let mut collection = ResourceCollection::::new(); - let resource = TestResource(Arc::new(1)); - - let rc_clone = collection.clone(); - - collection.on_resource_wired(); - - assert_matches!( - rc_clone.push(resource), - Err(ResourceCollectionError::AlreadyWired) - ); - } - - #[test] - fn test_resolve() { - let mut collection = ResourceCollection::::new(); - let result = collection.clone().resolve().now_or_never(); - - assert!(result.is_none()); - - collection.on_resource_wired(); - - let resolved = collection.resolve().now_or_never(); - assert_eq!(resolved.unwrap(), vec![]); - } -} diff --git a/core/node/node_framework/src/resource/unique.rs b/core/node/node_framework/src/resource/unique.rs index 9a256d8f55f3..5c9bdcfe0e12 100644 --- a/core/node/node_framework/src/resource/unique.rs +++ b/core/node/node_framework/src/resource/unique.rs @@ -29,6 +29,7 @@ impl Unique { } /// Takes the resource from the container. + /// Will return `None` if the resource was already taken. pub fn take(&self) -> Option { let result = self.inner.lock().unwrap().take(); diff --git a/core/node/node_framework/src/service/context.rs b/core/node/node_framework/src/service/context.rs index 9507c2287752..d4bb4db95464 100644 --- a/core/node/node_framework/src/service/context.rs +++ b/core/node/node_framework/src/service/context.rs @@ -3,15 +3,16 @@ use std::{any::type_name, future::Future}; use futures::FutureExt as _; use crate::{ - precondition::Precondition, resource::{Resource, ResourceId, StoredResource}, service::{named_future::NamedFuture, ZkStackService}, - task::{OneshotTask, Task, UnconstrainedOneshotTask, UnconstrainedTask}, + task::Task, wiring_layer::WiringError, }; -/// An interface to the service's resources provided to the tasks during initialization. -/// Provides the ability to fetch required resources, and also gives access to the Tokio runtime handle. +/// An interface to the service provided to the tasks during initialization. +/// This the main point of interaction between with the service. +/// +/// The context provides access to the runtime, resources, and allows adding new tasks. #[derive(Debug)] pub struct ServiceContext<'a> { layer: &'a str, @@ -19,16 +20,26 @@ pub struct ServiceContext<'a> { } impl<'a> ServiceContext<'a> { + /// Instantiates a new context. + /// The context keeps information about the layer that created it for reporting purposes. pub(super) fn new(layer: &'a str, service: &'a mut ZkStackService) -> Self { Self { layer, service } } /// Provides access to the runtime used by the service. + /// /// Can be used to spawn additional tasks within the same runtime. /// If some tasks stores the handle to spawn additional tasks, it is expected to do all the required /// cleanup. /// - /// In most cases, however, it is recommended to use [`add_task`] method instead. + /// In most cases, however, it is recommended to use [`add_task`](ServiceContext::add_task) or its alternative + /// instead. + /// + /// ## Note + /// + /// While `tokio::spawn` and `tokio::spawn_blocking` will work as well, using the runtime handle + /// from the context is still a recommended way to get access to runtime, as it tracks the access + /// to the runtimes by layers. pub fn runtime_handle(&self) -> &tokio::runtime::Handle { tracing::info!( "Layer {} has requested access to the Tokio runtime", @@ -38,6 +49,7 @@ impl<'a> ServiceContext<'a> { } /// Adds a task to the service. + /// /// Added tasks will be launched after the wiring process will be finished and all the preconditions /// are met. pub fn add_task(&mut self, task: Box) -> &mut Self { @@ -46,57 +58,6 @@ impl<'a> ServiceContext<'a> { self } - /// Adds an unconstrained task to the service. - /// Unconstrained tasks will be launched immediately after the wiring process is finished. - pub fn add_unconstrained_task(&mut self, task: Box) -> &mut Self { - tracing::info!( - "Layer {} has added a new unconstrained task: {}", - self.layer, - task.id() - ); - self.service.runnables.unconstrained_tasks.push(task); - self - } - - /// Adds a precondition to the service. - pub fn add_precondition(&mut self, precondition: Box) -> &mut Self { - tracing::info!( - "Layer {} has added a new precondition: {}", - self.layer, - precondition.id() - ); - self.service.runnables.preconditions.push(precondition); - self - } - - /// Adds an oneshot task to the service. - pub fn add_oneshot_task(&mut self, task: Box) -> &mut Self { - tracing::info!( - "Layer {} has added a new oneshot task: {}", - self.layer, - task.id() - ); - self.service.runnables.oneshot_tasks.push(task); - self - } - - /// Adds an unconstrained oneshot task to the service. - pub fn add_unconstrained_oneshot_task( - &mut self, - task: Box, - ) -> &mut Self { - tracing::info!( - "Layer {} has added a new unconstrained oneshot task: {}", - self.layer, - task.id() - ); - self.service - .runnables - .unconstrained_oneshot_tasks - .push(task); - self - } - /// Adds a future to be invoked after node shutdown. /// May be used to perform cleanup tasks. /// @@ -119,14 +80,15 @@ impl<'a> ServiceContext<'a> { self } - /// Attempts to retrieve the resource with the specified name. - /// Internally the resources are stored as [`std::any::Any`], and this method does the downcasting - /// on behalf of the caller. + /// Attempts to retrieve the resource of the specified type. /// /// ## Panics /// - /// Panics if the resource with the specified name exists, but is not of the requested type. + /// Panics if the resource with the specified [`ResourceId`] exists, but is not of the requested type. pub async fn get_resource(&mut self) -> Result { + // Implementation details: + // Internally the resources are stored as [`std::any::Any`], and this method does the downcasting + // on behalf of the caller. #[allow(clippy::borrowed_box)] let downcast_clone = |resource: &Box| { resource @@ -167,7 +129,7 @@ impl<'a> ServiceContext<'a> { }) } - /// Attempts to retrieve the resource with the specified name. + /// Attempts to retrieve the resource of the specified type. /// If the resource is not available, it is created using the provided closure. pub async fn get_resource_or_insert_with T>( &mut self, @@ -190,18 +152,19 @@ impl<'a> ServiceContext<'a> { resource } - /// Attempts to retrieve the resource with the specified name. + /// Attempts to retrieve the resource of the specified type. /// If the resource is not available, it is created using `T::default()`. pub async fn get_resource_or_default(&mut self) -> T { self.get_resource_or_insert_with(T::default).await } /// Adds a resource to the service. - /// If the resource with the same name is already provided, the method will return an error. + /// + /// If the resource with the same type is already provided, the method will return an error. pub fn insert_resource(&mut self, resource: T) -> Result<(), WiringError> { let id = ResourceId::of::(); if self.service.resources.contains_key(&id) { - tracing::warn!( + tracing::info!( "Layer {} has attempted to provide resource {} of type {}, but it is already available", self.layer, T::name(), diff --git a/core/node/node_framework/src/service/error.rs b/core/node/node_framework/src/service/error.rs index 9e95b437419b..890cc6b7d4b6 100644 --- a/core/node/node_framework/src/service/error.rs +++ b/core/node/node_framework/src/service/error.rs @@ -1,5 +1,6 @@ use crate::{task::TaskId, wiring_layer::WiringError}; +/// An error that can occur during the task lifecycle. #[derive(Debug, thiserror::Error)] pub enum TaskError { #[error("Task {0} failed: {1}")] @@ -14,6 +15,7 @@ pub enum TaskError { ShutdownHookTimedOut(TaskId), } +/// An error that can occur during the service lifecycle. #[derive(Debug, thiserror::Error)] pub enum ZkStackServiceError { #[error("Detected a Tokio Runtime. ZkStackService manages its own runtime and does not support nested runtimes")] diff --git a/core/node/node_framework/src/service/mod.rs b/core/node/node_framework/src/service/mod.rs index 57035a048d86..e727a536e9c4 100644 --- a/core/node/node_framework/src/service/mod.rs +++ b/core/node/node_framework/src/service/mod.rs @@ -1,17 +1,17 @@ use std::{collections::HashMap, time::Duration}; -use anyhow::Context; use error::TaskError; -use futures::FutureExt; -use runnables::NamedBoxFuture; -use tokio::{runtime::Runtime, sync::watch}; +use futures::future::Fuse; +use tokio::{runtime::Runtime, sync::watch, task::JoinHandle}; use zksync_utils::panic_extractor::try_extract_panic_message; -use self::runnables::Runnables; pub use self::{context::ServiceContext, error::ZkStackServiceError, stop_receiver::StopReceiver}; use crate::{ resource::{ResourceId, StoredResource}, - service::runnables::TaskReprs, + service::{ + named_future::NamedFuture, + runnables::{NamedBoxFuture, Runnables, TaskReprs}, + }, task::TaskId, wiring_layer::{WiringError, WiringLayer}, }; @@ -40,6 +40,7 @@ impl ZkStackServiceBuilder { } /// Adds a wiring layer. + /// /// During the [`run`](ZkStackService::run) call the service will invoke /// `wire` method of every layer in the order they were added. /// @@ -58,6 +59,10 @@ impl ZkStackServiceBuilder { self } + /// Builds the service. + /// + /// In case of errors during wiring phase, will return the list of all the errors that happened, in the order + /// of their occurrence. pub fn build(&mut self) -> Result { if tokio::runtime::Handle::try_current().is_ok() { return Err(ZkStackServiceError::RuntimeDetected); @@ -75,6 +80,7 @@ impl ZkStackServiceBuilder { runnables: Default::default(), stop_sender, runtime, + errors: Vec::new(), }) } } @@ -94,11 +100,38 @@ pub struct ZkStackService { stop_sender: watch::Sender, /// Tokio runtime used to spawn tasks. runtime: Runtime, + + /// Collector for the task errors met during the service execution. + errors: Vec, } +type TaskFuture = NamedFuture>>>; + impl ZkStackService { /// Runs the system. pub fn run(mut self) -> Result<(), ZkStackServiceError> { + self.wire()?; + + let TaskReprs { + tasks, + shutdown_hooks, + } = self.prepare_tasks(); + + let remaining = self.run_tasks(tasks); + self.shutdown_tasks(remaining); + self.run_shutdown_hooks(shutdown_hooks); + + tracing::info!("Exiting the service"); + if self.errors.is_empty() { + Ok(()) + } else { + Err(ZkStackServiceError::Task(self.errors)) + } + } + + /// Performs wiring of the service. + /// After invoking this method, the collected tasks will be collected in `self.runnables`. + fn wire(&mut self) -> Result<(), ZkStackServiceError> { // Initialize tasks. let wiring_layers = std::mem::take(&mut self.layers); @@ -108,8 +141,7 @@ impl ZkStackService { for layer in wiring_layers { let name = layer.layer_name().to_string(); // We must process wiring layers sequentially and in the same order as they were added. - let task_result = - runtime_handle.block_on(layer.wire(ServiceContext::new(&name, &mut self))); + let task_result = runtime_handle.block_on(layer.wire(ServiceContext::new(&name, self))); if let Err(err) = task_result { // We don't want to bail on the first error, since it'll provide worse DevEx: // People likely want to fix as much problems as they can in one go, rather than have @@ -131,43 +163,37 @@ impl ZkStackService { return Err(ZkStackServiceError::NoTasks); } - let only_oneshot_tasks = self.runnables.is_oneshot_only(); + // Wiring is now complete. + for resource in self.resources.values_mut() { + resource.stored_resource_wired(); + } + self.resources = HashMap::default(); // Decrement reference counters for resources. + tracing::info!("Wiring complete"); + + Ok(()) + } + /// Prepares collected tasks for running. + fn prepare_tasks(&mut self) -> TaskReprs { // Barrier that will only be lifted once all the preconditions are met. // It will be awaited by the tasks before they start running and by the preconditions once they are fulfilled. let task_barrier = self.runnables.task_barrier(); // Collect long-running tasks. let stop_receiver = StopReceiver(self.stop_sender.subscribe()); - let TaskReprs { - mut long_running_tasks, - oneshot_tasks, - shutdown_hooks, - } = self - .runnables - .prepare_tasks(task_barrier.clone(), stop_receiver.clone()); - - // Wiring is now complete. - for resource in self.resources.values_mut() { - resource.stored_resource_wired(); - } - drop(self.resources); // Decrement reference counters for resources. - tracing::info!("Wiring complete"); - - // Create a system task that is cancellation-aware and will only exit on either oneshot task failure or - // stop signal. - let oneshot_runner_system_task = - oneshot_runner_task(oneshot_tasks, stop_receiver, only_oneshot_tasks); - long_running_tasks.push(oneshot_runner_system_task); + self.runnables + .prepare_tasks(task_barrier.clone(), stop_receiver.clone()) + } + /// Spawn the provided tasks and runs them until at least one task exits, and returns the list + /// of remaining tasks. + /// Adds error, if any, to the `errors` vector. + fn run_tasks(&mut self, tasks: Vec>>) -> Vec { // Prepare tasks for running. let rt_handle = self.runtime.handle().clone(); - let join_handles: Vec<_> = long_running_tasks + let join_handles: Vec<_> = tasks .into_iter() - .map(|task| { - let name = task.id(); - NamedBoxFuture::new(rt_handle.spawn(task.into_inner()).fuse().boxed(), name) - }) + .map(|task| task.spawn(&rt_handle).fuse()) .collect(); // Collect names for remaining tasks for reporting purposes. @@ -179,11 +205,18 @@ impl ZkStackService { .block_on(futures::future::select_all(join_handles)); // Extract the result and report it to logs early, before waiting for any other task to shutdown. // We will also collect the errors from the remaining tasks, hence a vector. - let mut errors = Vec::new(); let task_name = tasks_names.swap_remove(resolved_idx); - handle_task_exit(resolved, task_name, &mut errors); + self.handle_task_exit(resolved, task_name); tracing::info!("One of the task has exited, shutting down the node"); + remaining + } + + /// Sends the stop signal and waits for the remaining tasks to finish. + fn shutdown_tasks(&mut self, remaining: Vec) { + // Send stop signal to remaining tasks and wait for them to finish. + self.stop_sender.send(true).ok(); + // Collect names for remaining tasks for reporting purposes. // We have to re-collect, becuase `select_all` does not guarantes the order of returned remaining futures. let remaining_tasks_names: Vec<_> = remaining.iter().map(|task| task.id()).collect(); @@ -192,8 +225,6 @@ impl ZkStackService { .map(|task| async { tokio::time::timeout(TASK_SHUTDOWN_TIMEOUT, task).await }) .collect(); - // Send stop signal to remaining tasks and wait for them to finish. - self.stop_sender.send(true).ok(); let execution_results = self .runtime .block_on(futures::future::join_all(remaining_tasks_with_timeout)); @@ -202,15 +233,18 @@ impl ZkStackService { for (name, result) in remaining_tasks_names.into_iter().zip(execution_results) { match result { Ok(resolved) => { - handle_task_exit(resolved, name, &mut errors); + self.handle_task_exit(resolved, name); } Err(_) => { tracing::error!("Task {name} timed out"); - errors.push(TaskError::TaskShutdownTimedOut(name)); + self.errors.push(TaskError::TaskShutdownTimedOut(name)); } } } + } + /// Runs the provided shutdown hooks. + fn run_shutdown_hooks(&mut self, shutdown_hooks: Vec>>) { // Run shutdown hooks sequentially. for hook in shutdown_hooks { let name = hook.id().clone(); @@ -223,86 +257,36 @@ impl ZkStackService { } Ok(Err(err)) => { tracing::error!("Shutdown hook {name} failed: {err}"); - errors.push(TaskError::ShutdownHookFailed(name, err)); + self.errors.push(TaskError::ShutdownHookFailed(name, err)); } Err(_) => { tracing::error!("Shutdown hook {name} timed out"); - errors.push(TaskError::ShutdownHookTimedOut(name)); + self.errors.push(TaskError::ShutdownHookTimedOut(name)); } } } - - tracing::info!("Exiting the service"); - if errors.is_empty() { - Ok(()) - } else { - Err(ZkStackServiceError::Task(errors)) - } } -} - -fn handle_task_exit( - task_result: Result, tokio::task::JoinError>, - task_name: TaskId, - errors: &mut Vec, -) { - match task_result { - Ok(Ok(())) => { - tracing::info!("Task {task_name} finished"); - } - Ok(Err(err)) => { - tracing::error!("Task {task_name} failed: {err}"); - errors.push(TaskError::TaskFailed(task_name, err)); - } - Err(panic_err) => { - let panic_msg = try_extract_panic_message(panic_err); - tracing::error!("Task {task_name} panicked: {panic_msg}"); - errors.push(TaskError::TaskPanicked(task_name, panic_msg)); - } - }; -} -fn oneshot_runner_task( - oneshot_tasks: Vec>>, - mut stop_receiver: StopReceiver, - only_oneshot_tasks: bool, -) -> NamedBoxFuture> { - let future = async move { - let oneshot_tasks = oneshot_tasks.into_iter().map(|fut| async move { - // Spawn each oneshot task as a separate tokio task. - // This way we can handle the cases when such a task panics and propagate the message - // to the service. - let handle = tokio::runtime::Handle::current(); - let name = fut.id().to_string(); - match handle.spawn(fut).await { - Ok(Ok(())) => Ok(()), - Ok(Err(err)) => Err(err).with_context(|| format!("Oneshot task {name} failed")), - Err(panic_err) => { - let panic_msg = try_extract_panic_message(panic_err); - Err(anyhow::format_err!( - "Oneshot task {name} panicked: {panic_msg}" - )) - } + /// Checks the result of the task execution, logs the result, and stores the error if any. + fn handle_task_exit( + &mut self, + task_result: Result, tokio::task::JoinError>, + task_name: TaskId, + ) { + match task_result { + Ok(Ok(())) => { + tracing::info!("Task {task_name} finished"); } - }); - - match futures::future::try_join_all(oneshot_tasks).await { - Err(err) => Err(err), - Ok(_) if only_oneshot_tasks => { - // We only run oneshot tasks in this service, so we can exit now. - Ok(()) + Ok(Err(err)) => { + tracing::error!("Task {task_name} failed: {err}"); + self.errors.push(TaskError::TaskFailed(task_name, err)); } - Ok(_) => { - // All oneshot tasks have exited and we have at least one long-running task. - // Simply wait for the stop signal. - stop_receiver.0.changed().await.ok(); - Ok(()) + Err(panic_err) => { + let panic_msg = try_extract_panic_message(panic_err); + tracing::error!("Task {task_name} panicked: {panic_msg}"); + self.errors + .push(TaskError::TaskPanicked(task_name, panic_msg)); } - } - // Note that we don't have to `select` on the stop signal explicitly: - // Each prerequisite is given a stop signal, and if everyone respects it, this future - // will still resolve once the stop signal is received. - }; - - NamedBoxFuture::new(future.boxed(), "oneshot_runner".into()) + }; + } } diff --git a/core/node/node_framework/src/service/named_future.rs b/core/node/node_framework/src/service/named_future.rs index 9aa715b0a74b..283fbbb327c9 100644 --- a/core/node/node_framework/src/service/named_future.rs +++ b/core/node/node_framework/src/service/named_future.rs @@ -1,6 +1,8 @@ use std::{fmt, future::Future, pin::Pin, task}; +use futures::future::{Fuse, FutureExt}; use pin_project_lite::pin_project; +use tokio::task::JoinHandle; use crate::task::TaskId; @@ -15,19 +17,34 @@ pin_project! { impl NamedFuture where - F: Future, + F: Future + Send + 'static, + F::Output: Send + 'static, { /// Creates a new future with the name tag attached. pub fn new(inner: F, name: TaskId) -> Self { Self { inner, name } } + /// Returns the ID of the task attached to the future. pub fn id(&self) -> TaskId { self.name.clone() } - pub fn into_inner(self) -> F { - self.inner + /// Fuses the wrapped future. + pub fn fuse(self) -> NamedFuture> { + NamedFuture { + name: self.name, + inner: self.inner.fuse(), + } + } + + /// Spawns the wrapped future on the provided runtime handle. + /// Returns a named wrapper over the join handle. + pub fn spawn(self, handle: &tokio::runtime::Handle) -> NamedFuture> { + NamedFuture { + name: self.name, + inner: handle.spawn(self.inner), + } } } diff --git a/core/node/node_framework/src/service/runnables.rs b/core/node/node_framework/src/service/runnables.rs index 8d240a8cffab..c3a7c21d2e80 100644 --- a/core/node/node_framework/src/service/runnables.rs +++ b/core/node/node_framework/src/service/runnables.rs @@ -1,30 +1,21 @@ use std::{fmt, sync::Arc}; -use futures::future::BoxFuture; +use anyhow::Context as _; +use futures::{future::BoxFuture, FutureExt as _}; use tokio::sync::Barrier; +use zksync_utils::panic_extractor::try_extract_panic_message; use super::{named_future::NamedFuture, StopReceiver}; -use crate::{ - precondition::Precondition, - task::{OneshotTask, Task, UnconstrainedOneshotTask, UnconstrainedTask}, -}; +use crate::task::{Task, TaskKind}; /// Alias for futures with the name assigned. -pub type NamedBoxFuture = NamedFuture>; +pub(crate) type NamedBoxFuture = NamedFuture>; /// A collection of different flavors of tasks. #[derive(Default)] pub(super) struct Runnables { - /// Preconditions added to the service. - pub(super) preconditions: Vec>, /// Tasks added to the service. pub(super) tasks: Vec>, - /// Oneshot tasks added to the service. - pub(super) oneshot_tasks: Vec>, - /// Unconstrained tasks added to the service. - pub(super) unconstrained_tasks: Vec>, - /// Unconstrained oneshot tasks added to the service. - pub(super) unconstrained_oneshot_tasks: Vec>, /// List of hooks to be invoked after node shutdown. pub(super) shutdown_hooks: Vec>>, } @@ -32,14 +23,7 @@ pub(super) struct Runnables { impl fmt::Debug for Runnables { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Runnables") - .field("preconditions", &self.preconditions) .field("tasks", &self.tasks) - .field("oneshot_tasks", &self.oneshot_tasks) - .field("unconstrained_tasks", &self.unconstrained_tasks) - .field( - "unconstrained_oneshot_tasks", - &self.unconstrained_oneshot_tasks, - ) .field("shutdown_hooks", &self.shutdown_hooks) .finish() } @@ -47,16 +31,14 @@ impl fmt::Debug for Runnables { /// A unified representation of tasks that can be run by the service. pub(super) struct TaskReprs { - pub(super) long_running_tasks: Vec>>, - pub(super) oneshot_tasks: Vec>>, + pub(super) tasks: Vec>>, pub(super) shutdown_hooks: Vec>>, } impl fmt::Debug for TaskReprs { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TaskReprs") - .field("long_running_tasks", &self.long_running_tasks.len()) - .field("oneshot_tasks", &self.oneshot_tasks.len()) + .field("long_running_tasks", &self.tasks.len()) .field("shutdown_hooks", &self.shutdown_hooks.len()) .finish() } @@ -68,130 +50,104 @@ impl Runnables { pub(super) fn is_empty(&self) -> bool { // We don't consider preconditions to be tasks. self.tasks.is_empty() - && self.oneshot_tasks.is_empty() - && self.unconstrained_tasks.is_empty() - && self.unconstrained_oneshot_tasks.is_empty() - } - - /// Returns `true` if there are no long-running tasks in the collection. - pub(super) fn is_oneshot_only(&self) -> bool { - self.tasks.is_empty() && self.unconstrained_tasks.is_empty() } /// Prepares a barrier that should be shared between tasks and preconditions. /// The barrier is configured to wait for all the participants to be ready. /// Barrier does not assume the existence of unconstrained tasks. pub(super) fn task_barrier(&self) -> Arc { - Arc::new(Barrier::new( - self.tasks.len() + self.preconditions.len() + self.oneshot_tasks.len(), - )) + let barrier_size = self + .tasks + .iter() + .filter(|t| { + matches!( + t.kind(), + TaskKind::Precondition | TaskKind::OneshotTask | TaskKind::Task + ) + }) + .count(); + Arc::new(Barrier::new(barrier_size)) } /// Transforms the collection of tasks into a set of universal futures. pub(super) fn prepare_tasks( - mut self, + &mut self, task_barrier: Arc, stop_receiver: StopReceiver, ) -> TaskReprs { let mut long_running_tasks = Vec::new(); - self.collect_unconstrained_tasks(&mut long_running_tasks, stop_receiver.clone()); - self.collect_tasks( - &mut long_running_tasks, - task_barrier.clone(), - stop_receiver.clone(), - ); - let mut oneshot_tasks = Vec::new(); - self.collect_preconditions( - &mut oneshot_tasks, - task_barrier.clone(), - stop_receiver.clone(), - ); - self.collect_oneshot_tasks( - &mut oneshot_tasks, - task_barrier.clone(), - stop_receiver.clone(), - ); - self.collect_unconstrained_oneshot_tasks(&mut oneshot_tasks, stop_receiver.clone()); - - TaskReprs { - long_running_tasks, - oneshot_tasks, - shutdown_hooks: self.shutdown_hooks, - } - } - fn collect_unconstrained_tasks( - &mut self, - tasks: &mut Vec>>, - stop_receiver: StopReceiver, - ) { - for task in std::mem::take(&mut self.unconstrained_tasks) { - let name = task.id(); - let stop_receiver = stop_receiver.clone(); - let task_future = Box::pin(task.run_unconstrained(stop_receiver)); - tasks.push(NamedFuture::new(task_future, name)); - } - } - - fn collect_tasks( - &mut self, - tasks: &mut Vec>>, - task_barrier: Arc, - stop_receiver: StopReceiver, - ) { for task in std::mem::take(&mut self.tasks) { let name = task.id(); + let kind = task.kind(); let stop_receiver = stop_receiver.clone(); let task_barrier = task_barrier.clone(); - let task_future = Box::pin(task.run_with_barrier(stop_receiver, task_barrier)); - tasks.push(NamedFuture::new(task_future, name)); + let task_future: BoxFuture<'static, _> = + Box::pin(task.run_internal(stop_receiver, task_barrier)); + let named_future = NamedFuture::new(task_future, name); + if kind.is_oneshot() { + oneshot_tasks.push(named_future); + } else { + long_running_tasks.push(named_future); + } } - } - fn collect_preconditions( - &mut self, - oneshot_tasks: &mut Vec>>, - task_barrier: Arc, - stop_receiver: StopReceiver, - ) { - for precondition in std::mem::take(&mut self.preconditions) { - let name = precondition.id(); - let stop_receiver = stop_receiver.clone(); - let task_barrier = task_barrier.clone(); - let task_future = - Box::pin(precondition.check_with_barrier(stop_receiver, task_barrier)); - oneshot_tasks.push(NamedFuture::new(task_future, name)); - } - } + let only_oneshot_tasks = long_running_tasks.is_empty(); + // Create a system task that is cancellation-aware and will only exit on either oneshot task failure or + // stop signal. + let oneshot_runner_system_task = + oneshot_runner_task(oneshot_tasks, stop_receiver, only_oneshot_tasks); + long_running_tasks.push(oneshot_runner_system_task); - fn collect_oneshot_tasks( - &mut self, - oneshot_tasks: &mut Vec>>, - task_barrier: Arc, - stop_receiver: StopReceiver, - ) { - for oneshot_task in std::mem::take(&mut self.oneshot_tasks) { - let name = oneshot_task.id(); - let stop_receiver = stop_receiver.clone(); - let task_barrier = task_barrier.clone(); - let task_future = - Box::pin(oneshot_task.run_oneshot_with_barrier(stop_receiver, task_barrier)); - oneshot_tasks.push(NamedFuture::new(task_future, name)); + TaskReprs { + tasks: long_running_tasks, + shutdown_hooks: std::mem::take(&mut self.shutdown_hooks), } } +} - fn collect_unconstrained_oneshot_tasks( - &mut self, - oneshot_tasks: &mut Vec>>, - stop_receiver: StopReceiver, - ) { - for unconstrained_oneshot_task in std::mem::take(&mut self.unconstrained_oneshot_tasks) { - let name = unconstrained_oneshot_task.id(); - let stop_receiver = stop_receiver.clone(); - let task_future = - Box::pin(unconstrained_oneshot_task.run_unconstrained_oneshot(stop_receiver)); - oneshot_tasks.push(NamedFuture::new(task_future, name)); +fn oneshot_runner_task( + oneshot_tasks: Vec>>, + mut stop_receiver: StopReceiver, + only_oneshot_tasks: bool, +) -> NamedBoxFuture> { + let future = async move { + let oneshot_tasks = oneshot_tasks.into_iter().map(|fut| async move { + // Spawn each oneshot task as a separate tokio task. + // This way we can handle the cases when such a task panics and propagate the message + // to the service. + let handle = tokio::runtime::Handle::current(); + let name = fut.id().to_string(); + match handle.spawn(fut).await { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) => Err(err).with_context(|| format!("Oneshot task {name} failed")), + Err(panic_err) => { + let panic_msg = try_extract_panic_message(panic_err); + Err(anyhow::format_err!( + "Oneshot task {name} panicked: {panic_msg}" + )) + } + } + }); + + match futures::future::try_join_all(oneshot_tasks).await { + Err(err) => Err(err), + Ok(_) if only_oneshot_tasks => { + // We only run oneshot tasks in this service, so we can exit now. + Ok(()) + } + Ok(_) => { + // All oneshot tasks have exited and we have at least one long-running task. + // Simply wait for the stop signal. + stop_receiver.0.changed().await.ok(); + Ok(()) + } } - } + // Note that we don't have to `select` on the stop signal explicitly: + // Each prerequisite is given a stop signal, and if everyone respects it, this future + // will still resolve once the stop signal is received. + }; + + NamedBoxFuture::new(future.boxed(), "oneshot_runner".into()) } diff --git a/core/node/node_framework/src/service/stop_receiver.rs b/core/node/node_framework/src/service/stop_receiver.rs index 7a181b49a80d..e174cf62ba36 100644 --- a/core/node/node_framework/src/service/stop_receiver.rs +++ b/core/node/node_framework/src/service/stop_receiver.rs @@ -8,9 +8,3 @@ use tokio::sync::watch; /// and prevent tasks from hanging by accident. #[derive(Debug, Clone)] pub struct StopReceiver(pub watch::Receiver); - -impl StopReceiver { - pub fn new(receiver: watch::Receiver) -> Self { - Self(receiver) - } -} diff --git a/core/node/node_framework/src/task.rs b/core/node/node_framework/src/task.rs deleted file mode 100644 index 8bb7bbd2c702..000000000000 --- a/core/node/node_framework/src/task.rs +++ /dev/null @@ -1,221 +0,0 @@ -//! Tasks define the "runnable" concept of the service, e.g. a unit of work that can be executed by the service. -//! -//! ## Task kinds -//! -//! This module defines different flavors of tasks. -//! The most basic one is [`Task`], which is only launched after all the preconditions are met (more on this later), -//! and is expected to run until the node is shut down. This is the most common type of task, e.g. API server, -//! state keeper, and metadata calculator are examples of such tasks. -//! -//! Then there exists an [`OneshotTask`], which has a clear exit condition that does not cause the node to shut down. -//! This is useful for tasks that are expected to run once and then exit, e.g. a task that performs a programmatic -//! migration. -//! -//! Finally, the task can be unconstrained by preconditions, which means that it will start immediately without -//! waiting for any preconditions to be met. This kind of tasks is represent by [`UnconstrainedTask`] and -//! [`UnconstrainedOneshotTask`]. -//! -//! ## Tasks and preconditions -//! -//! Besides tasks, service also has a concept of preconditions(crate::precondition::Precondition). Precondition is a -//! piece of logic that is expected to be met before the task can start. One can think of preconditions as a way to -//! express invariants that the tasks may rely on. -//! -//! In this notion, the difference between a task and an unconstrained task is that the former has all the invariants -//! checked already, and unrestricted task is responsible for *manually checking any invariants it may rely on*. -//! -//! The unrestricted tasks are rarely needed, but two common cases for them are: -//! - A task that must be started as soon as possible, e.g. healthcheck server. -//! - A task that may be a driving force for some precondition to be met. - -use std::{ - fmt::{self, Display, Formatter}, - ops::Deref, - sync::Arc, -}; - -use tokio::sync::Barrier; - -use crate::service::StopReceiver; - -/// A unique human-readable identifier of a task. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TaskId(String); - -impl TaskId { - pub fn new(value: String) -> Self { - TaskId(value) - } -} - -impl Display for TaskId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -impl From<&str> for TaskId { - fn from(value: &str) -> Self { - TaskId(value.to_owned()) - } -} - -impl From for TaskId { - fn from(value: String) -> Self { - TaskId(value) - } -} - -impl Deref for TaskId { - type Target = str; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -/// A task implementation. -/// -/// Note: any `Task` added to the service will only start after all the [preconditions](crate::precondition::Precondition) -/// are met. If a task should start immediately, one should use [`UnconstrainedTask`](crate::task::UnconstrainedTask). -#[async_trait::async_trait] -pub trait Task: 'static + Send { - /// Unique name of the task. - fn id(&self) -> TaskId; - - /// Runs the task. - /// - /// Once any of the task returns, the node will shutdown. - /// If the task returns an error, the node will spawn an error-level log message and will return a non-zero - /// exit code. - /// - /// `stop_receiver` argument contains a channel receiver that will change its value once the node requests - /// a shutdown. Every task is expected to either await or periodically check the state of channel and stop - /// its execution once the channel is changed. - /// - /// Each task is expected to perform the required cleanup after receiving the stop signal. - async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()>; -} - -impl dyn Task { - /// An internal helper method that guards running the task with a tokio Barrier. - /// Used to make sure that the task is not started until all the preconditions are met. - pub(super) async fn run_with_barrier( - self: Box, - mut stop_receiver: StopReceiver, - preconditions_barrier: Arc, - ) -> anyhow::Result<()> { - // Wait either for barrier to be lifted or for the stop signal to be received. - tokio::select! { - _ = preconditions_barrier.wait() => { - self.run(stop_receiver).await - } - _ = stop_receiver.0.changed() => { - Ok(()) - } - } - } -} - -impl fmt::Debug for dyn Task { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("Task").field("name", &self.id()).finish() - } -} - -/// A oneshot task implementation. -/// The difference from [`Task`] is that this kind of task may exit without causing the service to shutdown. -/// -/// Note: any `Task` added to the service will only start after all the [preconditions](crate::precondition::Precondition) -/// are met. If a task should start immediately, one should use [`UnconstrainedTask`](crate::task::UnconstrainedTask). -#[async_trait::async_trait] -pub trait OneshotTask: 'static + Send { - /// Unique name of the task. - fn id(&self) -> TaskId; - - /// Runs the task. - /// - /// Unlike [`Task::run`], this method is expected to return once the task is finished, without causing the - /// node to shutdown. - /// - /// `stop_receiver` argument contains a channel receiver that will change its value once the node requests - /// a shutdown. Every task is expected to either await or periodically check the state of channel and stop - /// its execution once the channel is changed. - /// - /// Each task is expected to perform the required cleanup after receiving the stop signal. - async fn run_oneshot(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()>; -} - -impl dyn OneshotTask { - /// An internal helper method that guards running the task with a tokio Barrier. - /// Used to make sure that the task is not started until all the preconditions are met. - pub(super) async fn run_oneshot_with_barrier( - self: Box, - mut stop_receiver: StopReceiver, - preconditions_barrier: Arc, - ) -> anyhow::Result<()> { - // Wait either for barrier to be lifted or for the stop signal to be received. - tokio::select! { - _ = preconditions_barrier.wait() => { - self.run_oneshot(stop_receiver).await - } - _ = stop_receiver.0.changed() => { - Ok(()) - } - } - } -} - -impl fmt::Debug for dyn OneshotTask { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OneshotTask") - .field("name", &self.id()) - .finish() - } -} - -/// A task implementation that is not constrained by preconditions. -/// -/// This trait is used to define tasks that should start immediately after the wiring phase, without waiting for -/// any preconditions to be met. -/// -/// *Warning*. An unconstrained task may not be aware of the state of the node and is expected to cautiously check -/// any invariants it may rely on. -#[async_trait::async_trait] -pub trait UnconstrainedTask: 'static + Send { - /// Unique name of the task. - fn id(&self) -> TaskId; - - /// Runs the task without waiting for any precondition to be met. - async fn run_unconstrained(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()>; -} - -impl fmt::Debug for dyn UnconstrainedTask { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("UnconstrainedTask") - .field("name", &self.id()) - .finish() - } -} - -/// An unconstrained analog of [`OneshotTask`]. -/// See [`UnconstrainedTask`] and [`OneshotTask`] for more details. -#[async_trait::async_trait] -pub trait UnconstrainedOneshotTask: 'static + Send { - /// Unique name of the task. - fn id(&self) -> TaskId; - - /// Runs the task without waiting for any precondition to be met. - async fn run_unconstrained_oneshot( - self: Box, - stop_receiver: StopReceiver, - ) -> anyhow::Result<()>; -} - -impl fmt::Debug for dyn UnconstrainedOneshotTask { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("UnconstrainedOneshotTask") - .field("name", &self.id()) - .finish() - } -} diff --git a/core/node/node_framework/src/task/mod.rs b/core/node/node_framework/src/task/mod.rs new file mode 100644 index 000000000000..8113a751441a --- /dev/null +++ b/core/node/node_framework/src/task/mod.rs @@ -0,0 +1,138 @@ +//! Tasks define the "runnable" concept of the service, e.g. a unit of work that can be executed by the service. + +use std::{ + fmt::{self, Formatter}, + sync::Arc, +}; + +use tokio::sync::Barrier; + +pub use self::types::{TaskId, TaskKind}; +use crate::service::StopReceiver; + +mod types; + +/// A task implementation. +/// Task defines the "runnable" concept of the service, e.g. a unit of work that can be executed by the service. +/// +/// Based on the task kind, the implemenation will be treated differently by the service. +/// +/// ## Task kinds +/// +/// There may be different kinds of tasks: +/// +/// ### `Task` +/// +/// A regular task. Returning from this task will cause the service to stop. [`Task::kind`] has a default +/// implementation that returns `TaskKind::Task`. +/// +/// Typically, the implementation of [`Task::run`] will be some form of loop that runs until either an +/// irrecoverable error happens (then task should return an error), or stop signal is received (then task should +/// return `Ok(())`). +/// +/// ### `OneshotTask` +/// +/// A task that can exit when completed without causing the service to terminate. +/// In case of `OneshotTask`s, the service will only exit when all the `OneshotTask`s have exited and there are +/// no more tasks running. +/// +/// ### `Precondition` +/// +/// A "barrier" task that is supposed to check invariants before the main tasks are started. +/// An example of a precondition task could be a task that checks if the database has all the required data. +/// Precondition tasks are often paired with some other kind of task that will make sure that the precondition +/// can be satisfied. This is required for a distributed service setup, where the precondition task will be +/// present on all the nodes, while a task that satisfies the precondition will be present only on one node. +/// +/// ### `UnconstrainedTask` +/// +/// A task that can run without waiting for preconditions. +/// Tasks of this kind are expected to check all the invariants they rely on themselves. +/// Usually, this kind of task is used either for tasks that must start as early as possible (e.g. healthcheck server), +/// or for tasks that cannot rely on preconditions. +/// +/// ### `UnconstrainedOneshotTask` +/// +/// A task that can run without waiting for preconditions and can exit without stopping the service. +/// Usually such tasks may be used for satisfying a precondition, for example, they can perform the database +/// setup. +#[async_trait::async_trait] +pub trait Task: 'static + Send { + /// Returns the kind of the task. + /// The returned values is expected to be static, and it will be used by the service + /// to determine how to handle the task. + fn kind(&self) -> TaskKind { + TaskKind::Task + } + + /// Unique name of the task. + fn id(&self) -> TaskId; + + /// Runs the task. + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()>; +} + +impl dyn Task { + /// An internal helper method that guards running the task with a tokio Barrier. + /// Used to make sure that the task is not started until all the preconditions are met. + pub(super) async fn run_internal( + self: Box, + stop_receiver: StopReceiver, + preconditions_barrier: Arc, + ) -> anyhow::Result<()> { + match self.kind() { + TaskKind::Task | TaskKind::OneshotTask => { + self.run_with_barrier(stop_receiver, preconditions_barrier) + .await + } + TaskKind::UnconstrainedTask | TaskKind::UnconstrainedOneshotTask => { + self.run(stop_receiver).await + } + TaskKind::Precondition => { + self.check_precondition(stop_receiver, preconditions_barrier) + .await + } + } + } + + async fn run_with_barrier( + self: Box, + mut stop_receiver: StopReceiver, + preconditions_barrier: Arc, + ) -> anyhow::Result<()> { + // Wait either for barrier to be lifted or for the stop signal to be received. + tokio::select! { + _ = preconditions_barrier.wait() => { + self.run(stop_receiver).await + } + _ = stop_receiver.0.changed() => { + Ok(()) + } + } + } + + async fn check_precondition( + self: Box, + mut stop_receiver: StopReceiver, + preconditions_barrier: Arc, + ) -> anyhow::Result<()> { + self.run(stop_receiver.clone()).await?; + tokio::select! { + _ = preconditions_barrier.wait() => { + Ok(()) + } + _ = stop_receiver.0.changed() => { + Ok(()) + } + } + } +} + +impl fmt::Debug for dyn Task { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("Task") + .field("kind", &self.kind()) + .field("name", &self.id()) + .finish() + } +} diff --git a/core/node/node_framework/src/task/types.rs b/core/node/node_framework/src/task/types.rs new file mode 100644 index 000000000000..70df61e56989 --- /dev/null +++ b/core/node/node_framework/src/task/types.rs @@ -0,0 +1,60 @@ +use std::{ + fmt::{Display, Formatter}, + ops::Deref, +}; + +/// Task kind. +/// See [`Task`](super::Task) documentation for more details. +#[derive(Debug, Clone, Copy)] +pub enum TaskKind { + Task, + OneshotTask, + UnconstrainedTask, + UnconstrainedOneshotTask, + Precondition, +} + +impl TaskKind { + pub(crate) fn is_oneshot(self) -> bool { + matches!( + self, + TaskKind::OneshotTask | TaskKind::UnconstrainedOneshotTask | TaskKind::Precondition + ) + } +} + +/// A unique human-readable identifier of a task. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TaskId(String); + +impl TaskId { + pub fn new(value: String) -> Self { + TaskId(value) + } +} + +impl Display for TaskId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl From<&str> for TaskId { + fn from(value: &str) -> Self { + TaskId(value.to_owned()) + } +} + +impl From for TaskId { + fn from(value: String) -> Self { + TaskId(value) + } +} + +impl Deref for TaskId { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} From 061097dcb8d1a152d7007605e10ee75f112447c2 Mon Sep 17 00:00:00 2001 From: Marcin M <128217157+mm-zk@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:29:29 +0200 Subject: [PATCH 2/6] chore: documentation about docker (#2328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ * Documentation explaining how to use docker to debug the CI issues. --- docs/guides/advanced/docker_and_ci.md | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docs/guides/advanced/docker_and_ci.md diff --git a/docs/guides/advanced/docker_and_ci.md b/docs/guides/advanced/docker_and_ci.md new file mode 100644 index 000000000000..ff1c7843b8b1 --- /dev/null +++ b/docs/guides/advanced/docker_and_ci.md @@ -0,0 +1,73 @@ +# Docker and CI + +How to efficiently debug CI issues locally. + +This document will be useful in case you struggle with reproducing some CI issues on your local machine. + +In most cases, this is due to the fact that your local machine has some arifacts, configs, files that you might have set +in the past, that are missing from the CI. + +## Basic docker commands + +- `docker ps` - prints the list of currently running containers +- `docker run` - starts a new docker container +- `docker exec` - connects to a running container and executes the command. +- `docker kill` - stops the container. +- `docker cp` - allows copying files between your system and docker container. + +Usually docker containers have a specific binary that they run, but for debugging we often want to start a bash instead. + +The command below starts a new docker containers, and instead of running its binary - runs `/bin/bash` in interactive +mode. + +``` +docker run -it matterlabs/zk-environment:latest2.0-lightweight-nightly /bin/bash +``` + +Connects to **already running** job, and gets you the interactive shell. + +``` +docker exec -i -it local-setup-zksync-1 /bin/bash +``` + +## Debugging CI + +Many of the tests require postgres & reth - you initialize them with: + +``` +docker compose up -d + +``` + +You should see something like this: + +``` +[+] Running 3/3 + ⠿ Network zksync-era_default Created 0.0s + ⠿ Container zksync-era-postgres-1 Started 0.3s + ⠿ Container zksync-era-reth-1 Started 0.3s +``` + +Start the docker with the 'basic' imge + +``` +# We tell it to connect to the same 'subnetwork' as other containers (zksync-era_default). +# the IN_DOCKER variable is changing different urls (like postgres) from localhost to postgres - so that it can connect to those +# containers above. +docker run --network zksync-era_default -e IN_DOCKER=1 -it matterlabs/zk-environment:latest2.0-lightweight-nightly /bin/bash +# and then inside, run: + +git clone https://github.com/matter-labs/zksync-era.git . +git checkout YOUR_BRANCH +zk +``` + +After this, you can run any commands you need. + +When you see a command like `ci_run zk contract build` in the CI - this simply means that it executed +`zk contract build` inside that docker container. + +**IMPORTANT** - by default, docker is running in the mode, where it does NOT persist the changes. So if you exit that +shell, all the changes will be removed (so when you restart, you'll end up in the same pristine condition). You can +'commit' your changes into a new docker image, using `docker commit XXX some_name`, where XXX is your container id from +`docker ps`. Afterwards you can 'start' this docker image with `docker run ... some_name`. From 6384cad26aead4d1bdbb606a97d623dacebf912c Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 26 Jun 2024 14:17:24 +0200 Subject: [PATCH 3/6] feat(zk toolbox): External node support (#2287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --------- Signed-off-by: Danil Co-authored-by: Matías Ignacio González --- .github/workflows/ci-zk-toolbox-reusable.yml | 19 ++- bin/zkt | 7 + chains/era/ZkStack.yaml | 1 + core/bin/external_node/src/config/mod.rs | 13 +- .../external_node/src/config/observability.rs | 11 +- core/bin/external_node/src/init.rs | 8 +- core/bin/external_node/src/main.rs | 1 + core/tests/ts-integration/src/env.ts | 24 +++- etc/env/file_based/general.yaml | 6 +- zk_toolbox/Cargo.lock | 1 + zk_toolbox/crates/config/src/chain.rs | 14 +- zk_toolbox/crates/config/src/consts.rs | 2 + zk_toolbox/crates/config/src/contracts.rs | 20 ++- zk_toolbox/crates/config/src/ecosystem.rs | 1 + zk_toolbox/crates/config/src/external_node.rs | 23 +++ zk_toolbox/crates/config/src/general.rs | 136 ++++++++++++++++++ zk_toolbox/crates/config/src/genesis.rs | 10 +- zk_toolbox/crates/config/src/lib.rs | 25 ++-- zk_toolbox/crates/config/src/secrets.rs | 19 ++- zk_toolbox/crates/config/src/traits.rs | 6 + .../zk_inception/src/accept_ownership.rs | 2 +- .../zk_inception/src/commands/args/mod.rs | 4 +- .../src/commands/args/run_server.rs | 2 +- .../src/commands/chain/args/genesis.rs | 6 +- .../zk_inception/src/commands/chain/create.rs | 1 + .../src/commands/chain/deploy_paymaster.rs | 14 +- .../src/commands/chain/genesis.rs | 31 ++-- .../zk_inception/src/commands/chain/init.rs | 67 ++++++--- .../src/commands/chain/initialize_bridges.rs | 21 ++- .../zk_inception/src/commands/chain/mod.rs | 14 +- .../src/commands/ecosystem/init.rs | 2 +- .../src/commands/external_node/args/mod.rs | 2 + .../external_node/args/prepare_configs.rs | 69 +++++++++ .../src/commands/external_node/args/run.rs | 15 ++ .../src/commands/external_node/init.rs | 53 +++++++ .../src/commands/external_node/mod.rs | 24 ++++ .../commands/external_node/prepare_configs.rs | 79 ++++++++++ .../src/commands/external_node/run.rs | 37 +++++ .../crates/zk_inception/src/commands/mod.rs | 1 + .../zk_inception/src/commands/server.rs | 17 +-- .../zk_inception/src/config_manipulations.rs | 97 ------------- zk_toolbox/crates/zk_inception/src/consts.rs | 2 + .../crates/zk_inception/src/defaults.rs | 14 +- .../crates/zk_inception/src/external_node.rs | 77 ++++++++++ zk_toolbox/crates/zk_inception/src/main.rs | 13 +- .../crates/zk_inception/src/messages.rs | 23 ++- zk_toolbox/crates/zk_inception/src/server.rs | 8 +- .../src/{forge_utils.rs => utils/forge.rs} | 0 .../crates/zk_inception/src/utils/mod.rs | 2 + .../crates/zk_inception/src/utils/rocks_db.rs | 39 +++++ zk_toolbox/crates/zk_supervisor/Cargo.toml | 1 + .../src/commands/integration_tests.rs | 46 ++++-- zk_toolbox/crates/zk_supervisor/src/dals.rs | 10 +- zk_toolbox/crates/zk_supervisor/src/main.rs | 8 +- .../crates/zk_supervisor/src/messages.rs | 19 ++- 55 files changed, 935 insertions(+), 232 deletions(-) create mode 100755 bin/zkt create mode 100644 zk_toolbox/crates/config/src/external_node.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs create mode 100644 zk_toolbox/crates/zk_inception/src/external_node.rs rename zk_toolbox/crates/zk_inception/src/{forge_utils.rs => utils/forge.rs} (100%) create mode 100644 zk_toolbox/crates/zk_inception/src/utils/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index 66e54bfa98a4..83ec7d1f5dc3 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -90,13 +90,30 @@ jobs: - name: Run server run: | - ci_run zk_inception server --ignore-prerequisites -a --verbose &>server.log & + ci_run zk_inception server --ignore-prerequisites &>server.log & ci_run sleep 5 - name: Run integration tests run: | ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose + + - name: Run external node server + run: | + ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \ + --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545 + ci_run zk_inception external-node init --ignore-prerequisites + ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log & + ci_run sleep 5 + + - name: Run integration tests en + run: | + ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose --external-node + - name: Show server.log logs if: always() run: ci_run cat server.log || true + - name: Show external_node.log logs + if: always() + run: ci_run cat external_node.log || true + diff --git a/bin/zkt b/bin/zkt new file mode 100755 index 000000000000..337ad5d73953 --- /dev/null +++ b/bin/zkt @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +cd $(dirname $0) +cd ../zk_toolbox + +cargo install --path ./crates/zk_inception --force +cargo install --path ./crates/zk_supervisor --force diff --git a/chains/era/ZkStack.yaml b/chains/era/ZkStack.yaml index 17b307cac4f6..8dbd49c02c67 100644 --- a/chains/era/ZkStack.yaml +++ b/chains/era/ZkStack.yaml @@ -4,6 +4,7 @@ chain_id: 271 prover_version: NoProofs configs: ./chains/era/configs/ rocks_db_path: ./chains/era/db/ +external_node_config_path: ./chains/era/configs/external_node l1_batch_commit_data_generator_mode: Rollup base_token: address: '0x0000000000000000000000000000000000000001' diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 35750cfa4e7d..b5b041a1fc6e 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -421,6 +421,9 @@ pub(crate) struct OptionalENConfig { #[serde(default = "OptionalENConfig::default_snapshots_recovery_postgres_max_concurrency")] pub snapshots_recovery_postgres_max_concurrency: NonZeroUsize, + #[serde(default)] + pub snapshot_recover_object_store: Option, + /// Enables pruning of the historical node state (Postgres and Merkle tree). The node will retain /// recent state and will continuously remove (prune) old enough parts of the state in the background. #[serde(default)] @@ -619,6 +622,10 @@ impl OptionalENConfig { .as_ref() .map(|a| a.enabled) .unwrap_or_default(), + snapshot_recover_object_store: load_config!( + general_config.snapshot_recovery, + object_store + ), pruning_chunk_size: load_optional_config_or_default!( general_config.pruning, chunk_size, @@ -798,9 +805,11 @@ impl OptionalENConfig { } fn from_env() -> anyhow::Result { - envy::prefixed("EN_") + let mut result: OptionalENConfig = envy::prefixed("EN_") .from_env() - .context("could not load external node config") + .context("could not load external node config")?; + result.snapshot_recover_object_store = snapshot_recovery_object_store_config().ok(); + Ok(result) } pub fn polling_interval(&self) -> Duration { diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index 39b86b8f0452..4dc310ee26c2 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -26,6 +26,8 @@ pub(crate) struct ObservabilityENConfig { /// Log format to use: either `plain` (default) or `json`. #[serde(default)] pub log_format: LogFormat, + // Log directives in format that is used in `RUST_LOG` + pub log_directives: Option, } impl ObservabilityENConfig { @@ -80,6 +82,9 @@ impl ObservabilityENConfig { pub fn build_observability(&self) -> anyhow::Result { let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(self.log_format); + if let Some(log_directives) = self.log_directives.clone() { + builder = builder.with_log_directives(log_directives) + }; // Some legacy deployments use `unset` as an equivalent of `None`. let sentry_url = self.sentry_url.as_deref().filter(|&url| url != "unset"); if let Some(sentry_url) = sentry_url { @@ -100,7 +105,7 @@ impl ObservabilityENConfig { } pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { - let (sentry_url, sentry_environment, log_format) = + let (sentry_url, sentry_environment, log_format, log_directives) = if let Some(observability) = general_config.observability.as_ref() { ( observability.sentry_url.clone(), @@ -109,9 +114,10 @@ impl ObservabilityENConfig { .log_format .parse() .context("Invalid log format")?, + observability.log_directives.clone(), ) } else { - (None, None, LogFormat::default()) + (None, None, LogFormat::default(), None) }; let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = if let Some(prometheus) = general_config.prometheus_config.as_ref() { @@ -130,6 +136,7 @@ impl ObservabilityENConfig { sentry_url, sentry_environment, log_format, + log_directives, }) } } diff --git a/core/bin/external_node/src/init.rs b/core/bin/external_node/src/init.rs index a9ee796194cc..ddf83a1f5581 100644 --- a/core/bin/external_node/src/init.rs +++ b/core/bin/external_node/src/init.rs @@ -3,6 +3,7 @@ use std::time::Instant; use anyhow::Context as _; +use zksync_config::ObjectStoreConfig; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_health_check::AppHealthCheck; use zksync_node_sync::genesis::perform_genesis_if_needed; @@ -12,12 +13,11 @@ use zksync_snapshots_applier::{SnapshotsApplierConfig, SnapshotsApplierTask}; use zksync_types::{L1BatchNumber, L2ChainId}; use zksync_web3_decl::client::{DynClient, L2}; -use crate::config::snapshot_recovery_object_store_config; - #[derive(Debug)] pub(crate) struct SnapshotRecoveryConfig { /// If not specified, the latest snapshot will be used. pub snapshot_l1_batch_override: Option, + pub object_store_config: Option, } #[derive(Debug)] @@ -90,7 +90,9 @@ pub(crate) async fn ensure_storage_initialized( )?; tracing::warn!("Proceeding with snapshot recovery. This is an experimental feature; use at your own risk"); - let object_store_config = snapshot_recovery_object_store_config()?; + let object_store_config = recovery_config.object_store_config.context( + "Snapshot object store must be presented if snapshot recovery is activated", + )?; let object_store = ObjectStoreFactory::new(object_store_config) .create_store() .await?; diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index c54bdc1dab19..0b3854b03c05 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -971,6 +971,7 @@ async fn run_node( .snapshots_recovery_enabled .then_some(SnapshotRecoveryConfig { snapshot_l1_batch_override: config.experimental.snapshots_recovery_l1_batch, + object_store_config: config.optional.snapshot_recover_object_store.clone(), }); ensure_storage_initialized( connection_pool.clone(), diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index c440e6b08ea6..ca97363fb4e2 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -57,11 +57,18 @@ function getMainWalletPk(pathToHome: string, network: string): string { */ async function loadTestEnvironmentFromFile(chain: string): Promise { const pathToHome = path.join(__dirname, '../../../..'); + let nodeMode; + if (process.env.EXTERNAL_NODE == 'true') { + nodeMode = NodeMode.External; + } else { + nodeMode = NodeMode.Main; + } let ecosystem = loadEcosystem(pathToHome); + // Genesis file is common for both EN and Main node + let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml', NodeMode.Main); - let generalConfig = loadConfig(pathToHome, chain, 'general.yaml'); - let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml'); - let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml'); + let generalConfig = loadConfig(pathToHome, chain, 'general.yaml', nodeMode); + let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml', nodeMode); const network = ecosystem.l1_network; let mainWalletPK = getMainWalletPk(pathToHome, network); @@ -115,8 +122,6 @@ async function loadTestEnvironmentFromFile(chain: string): Promise, pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, pub base_token: BaseToken, pub wallet_creation: WalletCreation, @@ -47,6 +49,7 @@ pub struct ChainConfig { pub link_to_code: PathBuf, pub rocks_db_path: PathBuf, pub configs: PathBuf, + pub external_node_config_path: Option, pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, pub base_token: BaseToken, pub wallet_creation: WalletCreation, @@ -71,6 +74,10 @@ impl ChainConfig { GenesisConfig::read(self.get_shell(), self.configs.join(GENESIS_FILE)) } + pub fn get_general_config(&self) -> anyhow::Result { + GeneralConfig::read(self.get_shell(), self.configs.join(GENERAL_FILE)) + } + pub fn get_wallets_config(&self) -> anyhow::Result { let path = self.configs.join(WALLETS_FILE); if let Ok(wallets) = WalletsConfig::read(self.get_shell(), &path) { @@ -100,7 +107,7 @@ impl ChainConfig { config.save(shell, path) } - pub fn save_with_base_path(&self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { + pub fn save_with_base_path(self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { let config = self.get_internal(); config.save_with_base_path(shell, path) } @@ -113,6 +120,7 @@ impl ChainConfig { prover_version: self.prover_version, configs: self.configs.clone(), rocks_db_path: self.rocks_db_path.clone(), + external_node_config_path: self.external_node_config_path.clone(), l1_batch_commit_data_generator_mode: self.l1_batch_commit_data_generator_mode, base_token: self.base_token.clone(), wallet_creation: self.wallet_creation, diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 9141d044af94..a00274fb35f3 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -11,6 +11,8 @@ pub(crate) const GENERAL_FILE: &str = "general.yaml"; /// Name of the genesis config file pub(crate) const GENESIS_FILE: &str = "genesis.yaml"; +// Name of external node specific config +pub(crate) const EN_CONFIG_FILE: &str = "external_node.yaml"; pub(crate) const ERC20_CONFIGS_FILE: &str = "erc20.yaml"; /// Name of the initial deployments config file pub(crate) const INITIAL_DEPLOYMENT_FILE: &str = "initial_deployments.yaml"; diff --git a/zk_toolbox/crates/config/src/contracts.rs b/zk_toolbox/crates/config/src/contracts.rs index b86b9b0f2958..a847c8a4cc93 100644 --- a/zk_toolbox/crates/config/src/contracts.rs +++ b/zk_toolbox/crates/config/src/contracts.rs @@ -3,7 +3,11 @@ use serde::{Deserialize, Serialize}; use crate::{ consts::CONTRACTS_FILE, - forge_interface::deploy_ecosystem::output::DeployL1Output, + forge_interface::{ + deploy_ecosystem::output::DeployL1Output, + initialize_bridges::output::InitializeBridgeOutput, + register_chain::output::RegisterChainOutput, + }, traits::{FileConfig, FileConfigWithDefaultName}, }; @@ -64,6 +68,20 @@ impl ContractsConfig { .diamond_cut_data .clone_from(&deploy_l1_output.contracts_config.diamond_cut_data); } + + pub fn set_chain_contracts(&mut self, register_chain_output: &RegisterChainOutput) { + self.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr; + self.l1.governance_addr = register_chain_output.governance_addr; + } + + pub fn set_l2_shared_bridge( + &mut self, + initialize_bridges_output: &InitializeBridgeOutput, + ) -> anyhow::Result<()> { + self.bridges.shared.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy); + self.bridges.erc20.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy); + Ok(()) + } } impl FileConfigWithDefaultName for ContractsConfig { diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index 1557ab21646f..08708ebb0b61 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -120,6 +120,7 @@ impl EcosystemConfig { chain_id: config.chain_id, prover_version: config.prover_version, configs: config.configs, + external_node_config_path: config.external_node_config_path, l1_batch_commit_data_generator_mode: config.l1_batch_commit_data_generator_mode, l1_network: self.l1_network, link_to_code: self diff --git a/zk_toolbox/crates/config/src/external_node.rs b/zk_toolbox/crates/config/src/external_node.rs new file mode 100644 index 000000000000..87acb15e4d8c --- /dev/null +++ b/zk_toolbox/crates/config/src/external_node.rs @@ -0,0 +1,23 @@ +use std::num::NonZeroUsize; + +use serde::{Deserialize, Serialize}; +use types::{ChainId, L1BatchCommitDataGeneratorMode}; + +use crate::{consts::EN_CONFIG_FILE, traits::FileConfigWithDefaultName}; + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct ENConfig { + // Genesis + pub l2_chain_id: ChainId, + pub l1_chain_id: u32, + pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, + + // Main node configuration + pub main_node_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub main_node_rate_limit_rps: Option, +} + +impl FileConfigWithDefaultName for ENConfig { + const FILE_NAME: &'static str = EN_CONFIG_FILE; +} diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 058f23bf1b5d..e1f3655d2200 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -1,17 +1,68 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; +use url::Url; use crate::{consts::GENERAL_FILE, traits::FileConfigWithDefaultName}; +pub struct RocksDbs { + pub state_keeper: PathBuf, + pub merkle_tree: PathBuf, +} + #[derive(Debug, Deserialize, Serialize, Clone)] pub struct GeneralConfig { pub db: RocksDBConfig, pub eth: EthConfig, + pub api: ApiConfig, #[serde(flatten)] pub other: serde_json::Value, } +impl GeneralConfig { + pub fn set_rocks_db_config(&mut self, rocks_dbs: RocksDbs) -> anyhow::Result<()> { + self.db.state_keeper_db_path = rocks_dbs.state_keeper; + self.db.merkle_tree.path = rocks_dbs.merkle_tree; + Ok(()) + } + + pub fn ports_config(&self) -> PortsConfig { + PortsConfig { + web3_json_rpc_http_port: self.api.web3_json_rpc.http_port, + web3_json_rpc_ws_port: self.api.web3_json_rpc.ws_port, + healthcheck_port: self.api.healthcheck.port, + merkle_tree_port: self.api.merkle_tree.port, + prometheus_listener_port: self.api.prometheus.listener_port, + } + } + + pub fn update_ports(&mut self, ports_config: &PortsConfig) -> anyhow::Result<()> { + self.api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port; + update_port_in_url( + &mut self.api.web3_json_rpc.http_url, + ports_config.web3_json_rpc_http_port, + )?; + self.api.web3_json_rpc.ws_port = ports_config.web3_json_rpc_ws_port; + update_port_in_url( + &mut self.api.web3_json_rpc.ws_url, + ports_config.web3_json_rpc_ws_port, + )?; + self.api.healthcheck.port = ports_config.healthcheck_port; + self.api.merkle_tree.port = ports_config.merkle_tree_port; + self.api.prometheus.listener_port = ports_config.prometheus_listener_port; + Ok(()) + } +} + +fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> { + let mut http_url_url = Url::parse(&http_url)?; + if let Err(()) = http_url_url.set_port(Some(port)) { + anyhow::bail!("Wrong url, setting port is impossible"); + } + *http_url = http_url_url.as_str().to_string(); + Ok(()) +} + impl FileConfigWithDefaultName for GeneralConfig { const FILE_NAME: &'static str = GENERAL_FILE; } @@ -45,3 +96,88 @@ pub struct EthSender { #[serde(flatten)] pub other: serde_json::Value, } + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct ApiConfig { + /// Configuration options for the Web3 JSON RPC servers. + pub web3_json_rpc: Web3JsonRpcConfig, + /// Configuration options for the Prometheus exporter. + pub prometheus: PrometheusConfig, + /// Configuration options for the Health check. + pub healthcheck: HealthCheckConfig, + /// Configuration options for Merkle tree API. + pub merkle_tree: MerkleTreeApiConfig, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct Web3JsonRpcConfig { + /// Port to which the HTTP RPC server is listening. + pub http_port: u16, + /// URL to access HTTP RPC server. + pub http_url: String, + /// Port to which the WebSocket RPC server is listening. + pub ws_port: u16, + /// URL to access WebSocket RPC server. + pub ws_url: String, + /// Max possible limit of entities to be requested once. + pub req_entities_limit: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct PrometheusConfig { + /// Port to which the Prometheus exporter server is listening. + pub listener_port: u16, + /// URL of the push gateway. + pub pushgateway_url: String, + /// Push interval in ms. + pub push_interval_ms: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct HealthCheckConfig { + /// Port to which the REST server is listening. + pub port: u16, + /// Time limit in milliseconds to mark a health check as slow and log the corresponding warning. + /// If not specified, the default value in the health check crate will be used. + pub slow_time_limit_ms: Option, + /// Time limit in milliseconds to abort a health check and return "not ready" status for the corresponding component. + /// If not specified, the default value in the health check crate will be used. + pub hard_time_limit_ms: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +/// Configuration for the Merkle tree API. +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct MerkleTreeApiConfig { + /// Port to bind the Merkle tree API server to. + pub port: u16, + #[serde(flatten)] + pub other: serde_json::Value, +} + +pub struct PortsConfig { + pub web3_json_rpc_http_port: u16, + pub web3_json_rpc_ws_port: u16, + pub healthcheck_port: u16, + pub merkle_tree_port: u16, + pub prometheus_listener_port: u16, +} + +impl PortsConfig { + pub fn next_empty_ports_config(&self) -> PortsConfig { + Self { + web3_json_rpc_http_port: self.web3_json_rpc_http_port + 100, + web3_json_rpc_ws_port: self.web3_json_rpc_ws_port + 100, + healthcheck_port: self.healthcheck_port + 100, + merkle_tree_port: self.merkle_tree_port + 100, + prometheus_listener_port: self.prometheus_listener_port + 100, + } + } +} diff --git a/zk_toolbox/crates/config/src/genesis.rs b/zk_toolbox/crates/config/src/genesis.rs index 4e3d931ea0f0..e666931870a8 100644 --- a/zk_toolbox/crates/config/src/genesis.rs +++ b/zk_toolbox/crates/config/src/genesis.rs @@ -2,7 +2,7 @@ use ethers::types::{Address, H256}; use serde::{Deserialize, Serialize}; use types::{ChainId, L1BatchCommitDataGeneratorMode, ProtocolSemanticVersion}; -use crate::{consts::GENESIS_FILE, traits::FileConfigWithDefaultName}; +use crate::{consts::GENESIS_FILE, traits::FileConfigWithDefaultName, ChainConfig}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct GenesisConfig { @@ -21,6 +21,14 @@ pub struct GenesisConfig { pub other: serde_json::Value, } +impl GenesisConfig { + pub fn update_from_chain_config(&mut self, config: &ChainConfig) { + self.l2_chain_id = config.chain_id; + self.l1_chain_id = config.l1_network.chain_id(); + self.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); + } +} + impl FileConfigWithDefaultName for GenesisConfig { const FILE_NAME: &'static str = GENESIS_FILE; } diff --git a/zk_toolbox/crates/config/src/lib.rs b/zk_toolbox/crates/config/src/lib.rs index 8e40da7bf6bd..a80a2b6fe5de 100644 --- a/zk_toolbox/crates/config/src/lib.rs +++ b/zk_toolbox/crates/config/src/lib.rs @@ -1,3 +1,15 @@ +pub use chain::*; +pub use consts::{DOCKER_COMPOSE_FILE, ZKSYNC_ERA_GIT_REPO}; +pub use contracts::*; +pub use ecosystem::*; +pub use file_config::*; +pub use general::*; +pub use genesis::*; +pub use manipulations::*; +pub use secrets::*; +pub use wallet_creation::*; +pub use wallets::*; + mod chain; mod consts; mod contracts; @@ -10,17 +22,6 @@ mod secrets; mod wallet_creation; mod wallets; +pub mod external_node; pub mod forge_interface; pub mod traits; - -pub use chain::*; -pub use consts::{DOCKER_COMPOSE_FILE, ZKSYNC_ERA_GIT_REPO}; -pub use contracts::*; -pub use ecosystem::*; -pub use file_config::*; -pub use general::*; -pub use genesis::*; -pub use manipulations::*; -pub use secrets::*; -pub use wallet_creation::*; -pub use wallets::*; diff --git a/zk_toolbox/crates/config/src/secrets.rs b/zk_toolbox/crates/config/src/secrets.rs index ebacc5d437cb..98a9be6ffe61 100644 --- a/zk_toolbox/crates/config/src/secrets.rs +++ b/zk_toolbox/crates/config/src/secrets.rs @@ -1,3 +1,4 @@ +use common::db::DatabaseConfig; use serde::{Deserialize, Serialize}; use url::Url; @@ -6,7 +7,8 @@ use crate::{consts::SECRETS_FILE, traits::FileConfigWithDefaultName}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DatabaseSecrets { pub server_url: Url, - pub prover_url: Url, + #[serde(skip_serializing_if = "Option::is_none")] + pub prover_url: Option, #[serde(flatten)] pub other: serde_json::Value, } @@ -26,6 +28,21 @@ pub struct SecretsConfig { pub other: serde_json::Value, } +impl SecretsConfig { + pub fn set_databases( + &mut self, + server_db_config: &DatabaseConfig, + prover_db_config: &DatabaseConfig, + ) { + self.database.server_url = server_db_config.full_url(); + self.database.prover_url = Some(prover_db_config.full_url()); + } + + pub fn set_l1_rpc_url(&mut self, l1_rpc_url: String) { + self.l1.l1_rpc_url = l1_rpc_url; + } +} + impl FileConfigWithDefaultName for SecretsConfig { const FILE_NAME: &'static str = SECRETS_FILE; } diff --git a/zk_toolbox/crates/config/src/traits.rs b/zk_toolbox/crates/config/src/traits.rs index 85c73e99f99b..79ae3a187a8b 100644 --- a/zk_toolbox/crates/config/src/traits.rs +++ b/zk_toolbox/crates/config/src/traits.rs @@ -18,11 +18,17 @@ pub trait FileConfigWithDefaultName { } impl FileConfig for T where T: FileConfigWithDefaultName {} + impl ReadConfig for T where T: FileConfig + Clone + DeserializeOwned {} + impl SaveConfig for T where T: FileConfig + Serialize {} + impl SaveConfigWithComment for T where T: FileConfig + Serialize {} + impl ReadConfigWithBasePath for T where T: FileConfigWithDefaultName + Clone + DeserializeOwned {} + impl SaveConfigWithBasePath for T where T: FileConfigWithDefaultName + Serialize {} + impl SaveConfigWithCommentAndBasePath for T where T: FileConfigWithDefaultName + Serialize {} /// Reads a config file from a given path, correctly parsing file extension. diff --git a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs index 830da513d4f0..179cb696ac3d 100644 --- a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs +++ b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs @@ -13,8 +13,8 @@ use ethers::types::{Address, H256}; use xshell::Shell; use crate::{ - forge_utils::{check_the_balance, fill_forge_private_key}, messages::MSG_ACCEPTING_GOVERNANCE_SPINNER, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn accept_admin( diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs index bf1457ba92c6..7b21015691b9 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs @@ -1,3 +1,3 @@ -mod run_server; - pub use run_server::*; + +mod run_server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs index 1ec211c25f6d..74bafd6ce5ef 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs @@ -13,5 +13,5 @@ pub struct RunServerArgs { pub genesis: bool, #[clap(long, short)] #[arg(trailing_var_arg = true, allow_hyphen_values = true, hide = false, help = MSG_SERVER_ADDITIONAL_ARGS_HELP)] - additional_args: Vec, + pub additional_args: Vec, } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs index 0b0529ea5139..483b78e9b267 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs @@ -9,8 +9,8 @@ use crate::{ defaults::{generate_db_names, DBNames, DATABASE_PROVER_URL, DATABASE_SERVER_URL}, messages::{ msg_prover_db_name_prompt, msg_prover_db_url_prompt, msg_server_db_name_prompt, - msg_server_db_url_prompt, MSG_GENESIS_USE_DEFAULT_HELP, MSG_PROVER_DB_NAME_HELP, - MSG_PROVER_DB_URL_HELP, MSG_SERVER_DB_NAME_HELP, MSG_SERVER_DB_URL_HELP, + msg_server_db_url_prompt, MSG_PROVER_DB_NAME_HELP, MSG_PROVER_DB_URL_HELP, + MSG_SERVER_DB_NAME_HELP, MSG_SERVER_DB_URL_HELP, MSG_USE_DEFAULT_DATABASES_HELP, }, }; @@ -24,7 +24,7 @@ pub struct GenesisArgs { pub prover_db_url: Option, #[clap(long, help = MSG_PROVER_DB_NAME_HELP)] pub prover_db_name: Option, - #[clap(long, short, help = MSG_GENESIS_USE_DEFAULT_HELP)] + #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] pub use_default: bool, #[clap(long, short, action)] pub dont_drop: bool, diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs index f915a3b8d6f6..dc8f408db3b3 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs @@ -68,6 +68,7 @@ pub(crate) fn create_chain_inner( link_to_code: ecosystem_config.link_to_code.clone(), rocks_db_path: ecosystem_config.get_chain_rocks_db_path(&default_chain_name), configs: chain_configs_path.clone(), + external_node_config_path: None, l1_batch_commit_data_generator_mode: args.l1_batch_commit_data_generator_mode, base_token: args.base_token, wallet_creation: args.wallet_creation, diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs index fe8dcdc562b2..4f82a92c2edc 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs @@ -9,15 +9,14 @@ use config::{ paymaster::{DeployPaymasterInput, DeployPaymasterOutput}, script_params::DEPLOY_PAYMASTER_SCRIPT_PARAMS, }, - traits::{ReadConfig, SaveConfig}, - ChainConfig, EcosystemConfig, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, }; use xshell::Shell; use crate::{ - config_manipulations::update_paymaster, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_PAYMASTER}, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { @@ -26,12 +25,15 @@ pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { let chain_config = ecosystem_config .load_chain(chain_name) .context(MSG_CHAIN_NOT_INITIALIZED)?; - deploy_paymaster(shell, &chain_config, args).await + let mut contracts = chain_config.get_contracts_config()?; + deploy_paymaster(shell, &chain_config, &mut contracts, args).await?; + contracts.save_with_base_path(shell, chain_config.configs) } pub async fn deploy_paymaster( shell: &Shell, chain_config: &ChainConfig, + contracts_config: &mut ContractsConfig, forge_args: ForgeScriptArgs, ) -> anyhow::Result<()> { let input = DeployPaymasterInput::new(chain_config)?; @@ -63,6 +65,6 @@ pub async fn deploy_paymaster( DEPLOY_PAYMASTER_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_paymaster(shell, chain_config, &output)?; + contracts_config.l2.testnet_paymaster_addr = output.paymaster; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 8c4edc88290d..554f9c2cf940 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -7,26 +7,25 @@ use common::{ logger, spinner::Spinner, }; -use config::{ChainConfig, EcosystemConfig}; +use config::{traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig}; +use types::ProverMode; use xshell::Shell; use super::args::genesis::GenesisArgsFinal; use crate::{ commands::chain::args::genesis::GenesisArgs, - config_manipulations::{update_database_secrets, update_general_config}, + consts::{PROVER_MIGRATIONS, SERVER_MIGRATIONS}, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_COMPLETED, MSG_INITIALIZING_DATABASES_SPINNER, MSG_INITIALIZING_PROVER_DATABASE, - MSG_INITIALIZING_SERVER_DATABASE, MSG_SELECTED_CONFIG, MSG_STARTING_GENESIS, - MSG_STARTING_GENESIS_SPINNER, + MSG_INITIALIZING_SERVER_DATABASE, MSG_RECREATE_ROCKS_DB_ERRROR, MSG_SELECTED_CONFIG, + MSG_STARTING_GENESIS, MSG_STARTING_GENESIS_SPINNER, }, server::{RunServer, ServerMode}, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; -const SERVER_MIGRATIONS: &str = "core/lib/dal/migrations"; -const PROVER_MIGRATIONS: &str = "prover/prover_dal/migrations"; - pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); let ecosystem_config = EcosystemConfig::from_file(shell)?; @@ -46,12 +45,20 @@ pub async fn genesis( shell: &Shell, config: &ChainConfig, ) -> anyhow::Result<()> { - // Clean the rocksdb - shell.remove_path(&config.rocks_db_path)?; shell.create_dir(&config.rocks_db_path)?; - update_general_config(shell, config)?; - update_database_secrets(shell, config, &args.server_db, &args.prover_db)?; + let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main) + .context(MSG_RECREATE_ROCKS_DB_ERRROR)?; + let mut general = config.get_general_config()?; + general.set_rocks_db_config(rocks_db)?; + if config.prover_version != ProverMode::NoProofs { + general.eth.sender.proof_sending_mode = "ONLY_REAL_PROOFS".to_string(); + } + general.save_with_base_path(shell, &config.configs)?; + + let mut secrets = config.get_secrets_config()?; + secrets.set_databases(&args.server_db, &args.prover_db); + secrets.save_with_base_path(&shell, &config.configs)?; logger::note( MSG_SELECTED_CONFIG, @@ -128,5 +135,5 @@ async fn initialize_databases( fn run_server_genesis(chain_config: &ChainConfig, shell: &Shell) -> anyhow::Result<()> { let server = RunServer::new(None, chain_config); - server.run(shell, ServerMode::Genesis) + server.run(shell, ServerMode::Genesis, vec![]) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 0c9ac8743eee..9660e30da15f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -1,5 +1,6 @@ use anyhow::Context; use common::{ + cmd::Cmd, config::global_config, forge::{Forge, ForgeScriptArgs}, logger, @@ -11,24 +12,25 @@ use config::{ register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, script_params::REGISTER_CHAIN_SCRIPT_PARAMS, }, - traits::{ReadConfig, ReadConfigWithBasePath, SaveConfig, SaveConfigWithBasePath}, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, ChainConfig, ContractsConfig, EcosystemConfig, }; -use xshell::Shell; +use xshell::{cmd, Shell}; -use super::args::init::InitArgsFinal; use crate::{ accept_ownership::accept_admin, commands::chain::{ - args::init::InitArgs, deploy_paymaster, genesis::genesis, initialize_bridges, + args::init::{InitArgs, InitArgsFinal}, + deploy_paymaster, + genesis::genesis, + initialize_bridges, }, - config_manipulations::{update_genesis, update_l1_contracts, update_l1_rpc_url_secret}, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{ - msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED, - MSG_CHAIN_NOT_FOUND_ERR, MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, + msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, + MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, }, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> { @@ -55,24 +57,32 @@ pub async fn init( chain_config: &ChainConfig, ) -> anyhow::Result<()> { copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?; + build_l1_contracts(shell, ecosystem_config)?; + + let mut genesis_config = chain_config.get_genesis_config()?; + genesis_config.update_from_chain_config(&chain_config); + genesis_config.save_with_base_path(shell, &chain_config.configs)?; - update_genesis(shell, chain_config)?; - update_l1_rpc_url_secret(shell, chain_config, init_args.l1_rpc_url.clone())?; - let mut contracts_config = - ContractsConfig::read_with_base_path(shell, &ecosystem_config.config)?; - contracts_config.l1.base_token_addr = chain_config.base_token.address; // Copy ecosystem contracts + let mut contracts_config = ecosystem_config.get_contracts_config()?; + contracts_config.l1.base_token_addr = chain_config.base_token.address; contracts_config.save_with_base_path(shell, &chain_config.configs)?; + let mut secrets = chain_config.get_secrets_config()?; + secrets.set_l1_rpc_url(init_args.l1_rpc_url.clone()); + secrets.save_with_base_path(shell, &chain_config.configs)?; + let spinner = Spinner::new(MSG_REGISTERING_CHAIN_SPINNER); - contracts_config = register_chain( + register_chain( shell, init_args.forge_args.clone(), ecosystem_config, chain_config, + &mut contracts_config, init_args.l1_rpc_url.clone(), ) .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); let spinner = Spinner::new(MSG_ACCEPTING_ADMIN_SPINNER); accept_admin( @@ -91,13 +101,21 @@ pub async fn init( shell, chain_config, ecosystem_config, + &mut contracts_config, init_args.forge_args.clone(), ) .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; if init_args.deploy_paymaster { - deploy_paymaster::deploy_paymaster(shell, chain_config, init_args.forge_args.clone()) - .await?; + deploy_paymaster::deploy_paymaster( + shell, + chain_config, + &mut contracts_config, + init_args.forge_args.clone(), + ) + .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; } genesis(init_args.genesis_args.clone(), shell, chain_config) @@ -112,13 +130,11 @@ async fn register_chain( forge_args: ForgeScriptArgs, config: &EcosystemConfig, chain_config: &ChainConfig, + contracts: &mut ContractsConfig, l1_rpc_url: String, -) -> anyhow::Result { +) -> anyhow::Result<()> { let deploy_config_path = REGISTER_CHAIN_SCRIPT_PARAMS.input(&config.link_to_code); - let contracts = config - .get_contracts_config() - .context(MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR)?; let deploy_config = RegisterChainL1Config::new(chain_config, &contracts)?; deploy_config.save(shell, deploy_config_path)?; @@ -136,5 +152,14 @@ async fn register_chain( shell, REGISTER_CHAIN_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_l1_contracts(shell, chain_config, ®ister_chain_output) + contracts.set_chain_contracts(®ister_chain_output); + Ok(()) +} + +fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { + let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry()); + let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS); + Cmd::new(cmd!(shell, "yarn build")).run()?; + spinner.finish(); + Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs index 4a81a2b26f1b..2fab4f8ae6d4 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs @@ -12,15 +12,14 @@ use config::{ initialize_bridges::{input::InitializeBridgeInput, output::InitializeBridgeOutput}, script_params::INITIALIZE_BRIDGES_SCRIPT_PARAMS, }, - traits::{ReadConfig, SaveConfig}, - ChainConfig, EcosystemConfig, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, }; use xshell::{cmd, Shell}; use crate::{ - config_manipulations::update_l2_shared_bridge, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_INITIALIZING_BRIDGES_SPINNER}, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { @@ -30,8 +29,17 @@ pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { .load_chain(chain_name) .context(MSG_CHAIN_NOT_INITIALIZED)?; + let mut contracts = chain_config.get_contracts_config()?; let spinner = Spinner::new(MSG_INITIALIZING_BRIDGES_SPINNER); - initialize_bridges(shell, &chain_config, &ecosystem_config, args).await?; + initialize_bridges( + shell, + &chain_config, + &ecosystem_config, + &mut contracts, + args, + ) + .await?; + contracts.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); Ok(()) @@ -41,6 +49,7 @@ pub async fn initialize_bridges( shell: &Shell, chain_config: &ChainConfig, ecosystem_config: &EcosystemConfig, + contracts_config: &mut ContractsConfig, forge_args: ForgeScriptArgs, ) -> anyhow::Result<()> { build_l2_contracts(shell, &ecosystem_config.link_to_code)?; @@ -74,7 +83,7 @@ pub async fn initialize_bridges( INITIALIZE_BRIDGES_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_l2_shared_bridge(shell, chain_config, &output)?; + contracts_config.set_l2_shared_bridge(&output)?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs index 759b4aaea557..aabb0d714c53 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs @@ -1,10 +1,3 @@ -pub(crate) mod args; -mod create; -pub mod deploy_paymaster; -pub mod genesis; -pub(crate) mod init; -mod initialize_bridges; - pub(crate) use args::create::ChainCreateArgsFinal; use clap::Subcommand; use common::forge::ForgeScriptArgs; @@ -13,6 +6,13 @@ use xshell::Shell; use crate::commands::chain::args::{create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs}; +pub(crate) mod args; +mod create; +pub mod deploy_paymaster; +pub mod genesis; +pub(crate) mod init; +mod initialize_bridges; + #[derive(Subcommand, Debug)] pub enum ChainCommands { /// Create a new chain, setting the necessary configurations for later initialization diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs index fecda40c7760..3099b3cf8c27 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs @@ -41,7 +41,6 @@ use crate::{ }, }, consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{ msg_ecosystem_initialized, msg_initializing_chain, MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, @@ -49,6 +48,7 @@ use crate::{ MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR, MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT, MSG_INITIALIZING_ECOSYSTEM, MSG_INTALLING_DEPS_SPINNER, }, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs new file mode 100644 index 000000000000..ebc7855c2b58 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs @@ -0,0 +1,2 @@ +pub mod prepare_configs; +pub mod run; diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs new file mode 100644 index 000000000000..e82fbd7ca155 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs @@ -0,0 +1,69 @@ +use clap::Parser; +use common::{db::DatabaseConfig, Prompt}; +use config::ChainConfig; +use serde::{Deserialize, Serialize}; +use slugify_rs::slugify; +use url::Url; + +use crate::{ + defaults::{generate_external_node_db_name, DATABASE_SERVER_URL, LOCAL_RPC_URL}, + messages::{ + msg_external_node_db_name_prompt, msg_external_node_db_url_prompt, MSG_L1_RPC_URL_PROMPT, + MSG_USE_DEFAULT_DATABASES_HELP, + }, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, Parser, Default)] +pub struct PrepareConfigArgs { + #[clap(long)] + pub db_url: Option, + #[clap(long)] + pub db_name: Option, + #[clap(long)] + pub l1_rpc_url: Option, + #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] + pub use_default: bool, +} + +impl PrepareConfigArgs { + pub fn fill_values_with_prompt(self, config: &ChainConfig) -> PrepareConfigFinal { + let db_name = generate_external_node_db_name(config); + let chain_name = config.name.clone(); + if self.use_default { + PrepareConfigFinal { + db: DatabaseConfig::new(DATABASE_SERVER_URL.clone(), db_name), + l1_rpc_url: LOCAL_RPC_URL.to_string(), + } + } else { + let db_url = self.db_url.unwrap_or_else(|| { + Prompt::new(&msg_external_node_db_url_prompt(&chain_name)) + .default(DATABASE_SERVER_URL.as_str()) + .ask() + }); + let db_name = slugify!( + &self.db_name.unwrap_or_else(|| { + Prompt::new(&msg_external_node_db_name_prompt(&chain_name)) + .default(&db_name) + .ask() + }), + separator = "_" + ); + let l1_rpc_url = self.l1_rpc_url.unwrap_or_else(|| { + Prompt::new(&MSG_L1_RPC_URL_PROMPT) + .default(&LOCAL_RPC_URL) + .ask() + }); + + PrepareConfigFinal { + db: DatabaseConfig::new(db_url, db_name), + l1_rpc_url, + } + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrepareConfigFinal { + pub db: DatabaseConfig, + pub l1_rpc_url: String, +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs new file mode 100644 index 000000000000..1bc0c06728d7 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs @@ -0,0 +1,15 @@ +use clap::Parser; +use serde::{Deserialize, Serialize}; + +use crate::messages::{MSG_SERVER_ADDITIONAL_ARGS_HELP, MSG_SERVER_COMPONENTS_HELP}; + +#[derive(Debug, Serialize, Deserialize, Parser)] +pub struct RunExternalNodeArgs { + #[clap(long)] + pub reinit: bool, + #[clap(long, help = MSG_SERVER_COMPONENTS_HELP)] + pub components: Option>, + #[clap(long, short)] + #[arg(trailing_var_arg = true, allow_hyphen_values = true, hide = false, help = MSG_SERVER_ADDITIONAL_ARGS_HELP)] + pub additional_args: Vec, +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs new file mode 100644 index 000000000000..c6101e88739c --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs @@ -0,0 +1,53 @@ +use anyhow::Context; +use common::{ + config::global_config, + db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, + spinner::Spinner, +}; +use config::{traits::ReadConfigWithBasePath, ChainConfig, EcosystemConfig, SecretsConfig}; +use xshell::Shell; + +use crate::{ + consts::SERVER_MIGRATIONS, + messages::{ + MSG_CHAIN_NOT_INITIALIZED, MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED, + MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_INITIALIZING_DATABASES_SPINNER, + }, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, +}; + +pub async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain = global_config().chain_name.clone(); + let chain_config = ecosystem_config + .load_chain(chain) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + init(shell, &chain_config).await +} + +pub async fn init(shell: &Shell, chain_config: &ChainConfig) -> anyhow::Result<()> { + let spin = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); + let secrets = SecretsConfig::read_with_base_path( + shell, + chain_config + .external_node_config_path + .clone() + .context(MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED)?, + )?; + let db_config = DatabaseConfig::from_url(secrets.database.server_url)?; + drop_db_if_exists(&db_config) + .await + .context(MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR)?; + init_db(&db_config).await?; + recreate_rocksdb_dirs( + shell, + &chain_config.rocks_db_path, + RocksDBDirOption::ExternalNode, + )?; + let path_to_server_migration = chain_config.link_to_code.join(SERVER_MIGRATIONS); + migrate_db(shell, path_to_server_migration, &db_config.full_url()).await?; + spin.finish(); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs new file mode 100644 index 000000000000..06e422de08b8 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs @@ -0,0 +1,24 @@ +use args::{prepare_configs::PrepareConfigArgs, run::RunExternalNodeArgs}; +use clap::Parser; +use serde::{Deserialize, Serialize}; +use xshell::Shell; + +mod args; +mod init; +mod prepare_configs; +mod run; + +#[derive(Debug, Serialize, Deserialize, Parser)] +pub enum ExternalNodeCommands { + Configs(PrepareConfigArgs), + Init, + Run(RunExternalNodeArgs), +} + +pub async fn run(shell: &Shell, commands: ExternalNodeCommands) -> anyhow::Result<()> { + match commands { + ExternalNodeCommands::Configs(args) => prepare_configs::run(shell, args), + ExternalNodeCommands::Init => init::run(shell).await, + ExternalNodeCommands::Run(args) => run::run(shell, args).await, + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs new file mode 100644 index 000000000000..4df420474ecb --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -0,0 +1,79 @@ +use std::path::Path; + +use anyhow::Context; +use common::{config::global_config, logger}; +use config::{ + external_node::ENConfig, traits::SaveConfigWithBasePath, ChainConfig, DatabaseSecrets, + EcosystemConfig, L1Secret, SecretsConfig, +}; +use xshell::Shell; + +use crate::{ + commands::external_node::args::prepare_configs::{PrepareConfigArgs, PrepareConfigFinal}, + messages::{ + msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, + }, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, +}; + +pub fn run(shell: &Shell, args: PrepareConfigArgs) -> anyhow::Result<()> { + logger::info(MSG_PREPARING_EN_CONFIGS); + let chain_name = global_config().chain_name.clone(); + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let mut chain_config = ecosystem_config + .load_chain(chain_name) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + let args = args.fill_values_with_prompt(&chain_config); + let external_node_config_path = chain_config + .external_node_config_path + .unwrap_or_else(|| chain_config.configs.join("external_node")); + shell.create_dir(&external_node_config_path)?; + chain_config.external_node_config_path = Some(external_node_config_path.clone()); + prepare_configs(shell, &chain_config, &external_node_config_path, args)?; + let chain_path = ecosystem_config.chains.join(&chain_config.name); + chain_config.save_with_base_path(shell, chain_path)?; + logger::info(msg_preparing_en_config_is_done(&external_node_config_path)); + Ok(()) +} + +fn prepare_configs( + shell: &Shell, + config: &ChainConfig, + en_configs_path: &Path, + args: PrepareConfigFinal, +) -> anyhow::Result<()> { + let genesis = config.get_genesis_config()?; + let general = config.get_general_config()?; + let en_config = ENConfig { + l2_chain_id: genesis.l2_chain_id, + l1_chain_id: genesis.l1_chain_id, + l1_batch_commit_data_generator_mode: genesis + .l1_batch_commit_data_generator_mode + .unwrap_or_default(), + main_node_url: general.api.web3_json_rpc.http_url.clone(), + main_node_rate_limit_rps: None, + }; + let mut general_en = general.clone(); + general_en.update_ports(&general.ports_config().next_empty_ports_config())?; + let secrets = SecretsConfig { + database: DatabaseSecrets { + server_url: args.db.full_url(), + prover_url: None, + other: Default::default(), + }, + l1: L1Secret { + l1_rpc_url: args.l1_rpc_url.clone(), + other: Default::default(), + }, + other: Default::default(), + }; + secrets.save_with_base_path(shell, en_configs_path)?; + let dirs = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::ExternalNode)?; + general_en.set_rocks_db_config(dirs)?; + + general_en.save_with_base_path(shell, &en_configs_path)?; + en_config.save_with_base_path(shell, &en_configs_path)?; + + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs new file mode 100644 index 000000000000..9d3da4663859 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs @@ -0,0 +1,37 @@ +use anyhow::Context; +use common::{config::global_config, logger}; +use config::{ChainConfig, EcosystemConfig}; +use xshell::Shell; + +use crate::{ + commands::external_node::{args::run::RunExternalNodeArgs, init}, + external_node::RunExternalNode, + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_EN}, +}; + +pub async fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain = global_config().chain_name.clone(); + let chain_config = ecosystem_config + .load_chain(chain) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + logger::info(MSG_STARTING_EN); + + run_external_node(args, &chain_config, shell).await?; + + Ok(()) +} + +async fn run_external_node( + args: RunExternalNodeArgs, + chain_config: &ChainConfig, + shell: &Shell, +) -> anyhow::Result<()> { + if args.reinit { + init::init(shell, chain_config).await? + } + let server = RunExternalNode::new(args.components.clone(), chain_config)?; + server.run(shell, args.additional_args.clone()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/mod.rs index ccdf5b082caa..db34e1d8647d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/mod.rs @@ -2,5 +2,6 @@ pub mod args; pub mod chain; pub mod containers; pub mod ecosystem; +pub mod external_node; pub mod prover; pub mod server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/server.rs b/zk_toolbox/crates/zk_inception/src/commands/server.rs index e2d35dd9b792..aed16357c925 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/server.rs @@ -1,11 +1,11 @@ use anyhow::Context; -use common::{cmd::Cmd, config::global_config, logger, spinner::Spinner}; +use common::{config::global_config, logger}; use config::{ChainConfig, EcosystemConfig}; -use xshell::{cmd, Shell}; +use xshell::Shell; use crate::{ commands::args::RunServerArgs, - messages::{MSG_BUILDING_L1_CONTRACTS, MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, server::{RunServer, ServerMode}, }; @@ -19,20 +19,11 @@ pub fn run(shell: &Shell, args: RunServerArgs) -> anyhow::Result<()> { logger::info(MSG_STARTING_SERVER); - build_l1_contracts(shell, &ecosystem_config)?; run_server(args, &chain_config, shell)?; Ok(()) } -fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { - let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry()); - let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS); - Cmd::new(cmd!(shell, "yarn build")).run()?; - spinner.finish(); - Ok(()) -} - fn run_server( args: RunServerArgs, chain_config: &ChainConfig, @@ -44,5 +35,5 @@ fn run_server( } else { ServerMode::Normal }; - server.run(shell, mode) + server.run(shell, mode, args.additional_args) } diff --git a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs index a300a15e76c6..e69de29bb2d1 100644 --- a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs +++ b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs @@ -1,97 +0,0 @@ -use common::db::DatabaseConfig; -use config::{ - forge_interface::{ - initialize_bridges::output::InitializeBridgeOutput, paymaster::DeployPaymasterOutput, - register_chain::output::RegisterChainOutput, - }, - traits::{ReadConfigWithBasePath, SaveConfigWithBasePath}, - ChainConfig, ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, -}; -use types::ProverMode; -use xshell::Shell; - -use crate::defaults::{ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE}; - -pub(crate) fn update_genesis(shell: &Shell, config: &ChainConfig) -> anyhow::Result<()> { - let mut genesis = GenesisConfig::read_with_base_path(shell, &config.configs)?; - - genesis.l2_chain_id = config.chain_id; - genesis.l1_chain_id = config.l1_network.chain_id(); - genesis.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); - - genesis.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_database_secrets( - shell: &Shell, - config: &ChainConfig, - server_db_config: &DatabaseConfig, - prover_db_config: &DatabaseConfig, -) -> anyhow::Result<()> { - let mut secrets = SecretsConfig::read_with_base_path(shell, &config.configs)?; - secrets.database.server_url = server_db_config.full_url(); - secrets.database.prover_url = prover_db_config.full_url(); - secrets.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_l1_rpc_url_secret( - shell: &Shell, - config: &ChainConfig, - l1_rpc_url: String, -) -> anyhow::Result<()> { - let mut secrets = SecretsConfig::read_with_base_path(shell, &config.configs)?; - secrets.l1.l1_rpc_url = l1_rpc_url; - secrets.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_general_config(shell: &Shell, config: &ChainConfig) -> anyhow::Result<()> { - let mut general = GeneralConfig::read_with_base_path(shell, &config.configs)?; - general.db.state_keeper_db_path = - shell.create_dir(config.rocks_db_path.join(ROCKS_DB_STATE_KEEPER))?; - general.db.merkle_tree.path = shell.create_dir(config.rocks_db_path.join(ROCKS_DB_TREE))?; - if config.prover_version != ProverMode::NoProofs { - general.eth.sender.proof_sending_mode = "ONLY_REAL_PROOFS".to_string(); - } - general.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub fn update_l1_contracts( - shell: &Shell, - config: &ChainConfig, - register_chain_output: &RegisterChainOutput, -) -> anyhow::Result { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr; - contracts_config.l1.governance_addr = register_chain_output.governance_addr; - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(contracts_config) -} - -pub fn update_l2_shared_bridge( - shell: &Shell, - config: &ChainConfig, - initialize_bridges_output: &InitializeBridgeOutput, -) -> anyhow::Result<()> { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.bridges.shared.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); - contracts_config.bridges.erc20.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub fn update_paymaster( - shell: &Shell, - config: &ChainConfig, - paymaster_output: &DeployPaymasterOutput, -) -> anyhow::Result<()> { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.l2.testnet_paymaster_addr = paymaster_output.paymaster; - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(()) -} diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index a59024d09b40..8dde9337a73f 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -1,3 +1,5 @@ pub const AMOUNT_FOR_DISTRIBUTION_TO_WALLETS: u128 = 1000000000000000000000; pub const MINIMUM_BALANCE_FOR_WALLET: u128 = 5000000000000000000; +pub const SERVER_MIGRATIONS: &str = "core/lib/dal/migrations"; +pub const PROVER_MIGRATIONS: &str = "prover/prover_dal/migrations"; diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 04b735e02275..40be1293614b 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -9,8 +9,10 @@ lazy_static! { Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap(); } -pub const ROCKS_DB_STATE_KEEPER: &str = "main/state_keeper"; -pub const ROCKS_DB_TREE: &str = "main/tree"; +pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper"; +pub const ROCKS_DB_TREE: &str = "tree"; +pub const EN_ROCKS_DB_PREFIX: &str = "en"; +pub const MAIN_ROCKS_DB_PREFIX: &str = "main"; pub const L2_CHAIN_ID: u32 = 271; /// Path to base chain configuration inside zksync-era @@ -36,3 +38,11 @@ pub fn generate_db_names(config: &ChainConfig) -> DBNames { ), } } + +pub fn generate_external_node_db_name(config: &ChainConfig) -> String { + format!( + "external_node_{}_{}", + config.l1_network.to_string().to_ascii_lowercase(), + config.name + ) +} diff --git a/zk_toolbox/crates/zk_inception/src/external_node.rs b/zk_toolbox/crates/zk_inception/src/external_node.rs new file mode 100644 index 000000000000..baf00cccae5f --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/external_node.rs @@ -0,0 +1,77 @@ +use std::path::PathBuf; + +use anyhow::Context; +use common::cmd::Cmd; +use config::{ + external_node::ENConfig, traits::FileConfigWithDefaultName, ChainConfig, GeneralConfig, + SecretsConfig, +}; +use xshell::{cmd, Shell}; + +use crate::messages::MSG_FAILED_TO_RUN_SERVER_ERR; + +pub struct RunExternalNode { + components: Option>, + code_path: PathBuf, + general_config: PathBuf, + secrets: PathBuf, + en_config: PathBuf, +} + +impl RunExternalNode { + pub fn new( + components: Option>, + chain_config: &ChainConfig, + ) -> anyhow::Result { + let en_path = chain_config + .external_node_config_path + .clone() + .context("External node is not initialized")?; + let general_config = GeneralConfig::get_path_with_base_path(&en_path); + let secrets = SecretsConfig::get_path_with_base_path(&en_path); + let enconfig = ENConfig::get_path_with_base_path(&en_path); + + Ok(Self { + components, + code_path: chain_config.link_to_code.clone(), + general_config, + secrets, + en_config: enconfig, + }) + } + + pub fn run(&self, shell: &Shell, mut additional_args: Vec) -> anyhow::Result<()> { + shell.change_dir(&self.code_path); + let config_general_config = &self.general_config.to_str().unwrap(); + let en_config = &self.en_config.to_str().unwrap(); + let secrets = &self.secrets.to_str().unwrap(); + if let Some(components) = self.components() { + additional_args.push(format!("--components={}", components)) + } + let mut cmd = Cmd::new( + cmd!( + shell, + "cargo run --release --bin zksync_external_node -- + --config-path {config_general_config} + --secrets-path {secrets} + --external-node-config-path {en_config} + " + ) + .args(additional_args) + .env_remove("RUSTUP_TOOLCHAIN"), + ) + .with_force_run(); + + cmd.run().context(MSG_FAILED_TO_RUN_SERVER_ERR)?; + Ok(()) + } + + fn components(&self) -> Option { + self.components.as_ref().and_then(|components| { + if components.is_empty() { + return None; + } + Some(components.join(",")) + }) + } +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index dff9e479e01f..f381ad7fb47c 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -8,17 +8,18 @@ use config::EcosystemConfig; use xshell::Shell; use crate::commands::{ - args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, prover::ProverCommands, + args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, + external_node::ExternalNodeCommands, prover::ProverCommands, }; pub mod accept_ownership; mod commands; -mod config_manipulations; mod consts; mod defaults; -pub mod forge_utils; +pub mod external_node; mod messages; pub mod server; +mod utils; #[derive(Parser, Debug)] #[command(version, about)] @@ -42,6 +43,9 @@ pub enum InceptionSubcommands { Prover(ProverCommands), /// Run server Server(RunServerArgs), + // Run External Node + #[command(subcommand)] + ExternalNode(ExternalNodeCommands), /// Run containers for local development Containers, } @@ -109,6 +113,9 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?, InceptionSubcommands::Server(args) => commands::server::run(shell, args)?, InceptionSubcommands::Containers => commands::containers::run(shell)?, + InceptionSubcommands::ExternalNode(args) => { + commands::external_node::run(shell, args).await? + } } Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 1b3c05258753..1fa36fbabb1b 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use ethers::{ types::{H160, U256}, utils::format_ether, @@ -43,7 +45,6 @@ pub(super) const MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT: &str = "Provide the path t pub(super) const MSG_L1_RPC_URL_INVALID_ERR: &str = "Invalid RPC URL"; pub(super) const MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR: &str = "Invalid path"; pub(super) const MSG_GENESIS_DATABASE_ERR: &str = "Unable to perform genesis on the database"; -pub(super) const MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR: &str = "Ecosystem contracts config not found"; pub(super) const MSG_CHAIN_NOT_FOUND_ERR: &str = "Chain not found"; pub(super) const MSG_INITIALIZING_ECOSYSTEM: &str = "Initializing ecosystem"; pub(super) const MSG_DEPLOYING_ERC20: &str = "Deploying ERC20 contracts"; @@ -55,6 +56,7 @@ pub(super) const MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER: &str = "Deploying ecosystem contracts..."; pub(super) const MSG_REGISTERING_CHAIN_SPINNER: &str = "Registering chain..."; pub(super) const MSG_ACCEPTING_ADMIN_SPINNER: &str = "Accepting admin..."; +pub(super) const MSG_RECREATE_ROCKS_DB_ERRROR: &str = "Failed to create rocks db path"; pub(super) fn msg_initializing_chain(chain_name: &str) -> String { format!("Initializing chain {chain_name}") @@ -118,7 +120,7 @@ pub(super) const MSG_SERVER_DB_URL_HELP: &str = "Server database url without dat pub(super) const MSG_SERVER_DB_NAME_HELP: &str = "Server database name"; pub(super) const MSG_PROVER_DB_URL_HELP: &str = "Prover database url without database name"; pub(super) const MSG_PROVER_DB_NAME_HELP: &str = "Prover database name"; -pub(super) const MSG_GENESIS_USE_DEFAULT_HELP: &str = "Use default database urls and names"; +pub(super) const MSG_USE_DEFAULT_DATABASES_HELP: &str = "Use default database urls and names"; pub(super) const MSG_GENESIS_COMPLETED: &str = "Genesis completed successfully"; pub(super) const MSG_STARTING_GENESIS: &str = "Starting genesis process"; pub(super) const MSG_INITIALIZING_DATABASES_SPINNER: &str = "Initializing databases..."; @@ -133,6 +135,10 @@ pub(super) fn msg_server_db_url_prompt(chain_name: &str) -> String { format!("Please provide server database url for chain {chain_name}") } +pub(super) fn msg_external_node_db_url_prompt(chain_name: &str) -> String { + format!("Please provide external_node database url for chain {chain_name}") +} + pub(super) fn msg_prover_db_url_prompt(chain_name: &str) -> String { format!("Please provide prover database url for chain {chain_name}") } @@ -141,6 +147,10 @@ pub(super) fn msg_prover_db_name_prompt(chain_name: &str) -> String { format!("Please provide prover database name for chain {chain_name}") } +pub(super) fn msg_external_node_db_name_prompt(chain_name: &str) -> String { + format!("Please provide external_node database name for chain {chain_name}") +} + pub(super) fn msg_server_db_name_prompt(chain_name: &str) -> String { format!("Please provide server database name for chain {chain_name}") } @@ -173,6 +183,7 @@ pub(super) const MSG_FAILED_TO_FIND_ECOSYSTEM_ERR: &str = "Failed to find ecosys pub(super) const MSG_STARTING_SERVER: &str = "Starting server"; pub(super) const MSG_FAILED_TO_RUN_SERVER_ERR: &str = "Failed to start server"; pub(super) const MSG_BUILDING_L1_CONTRACTS: &str = "Building L1 contracts..."; +pub(super) const MSG_PREPARING_EN_CONFIGS: &str = "Preparing External Node config"; /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; @@ -189,6 +200,14 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( ) } +pub(super) fn msg_preparing_en_config_is_done(path: &Path) -> String { + format!("External nodes configs could be found in: {path:?}") +} + +pub(super) const MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED: &str = + "External node is not initialized"; /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; + +pub(super) const MSG_STARTING_EN: &str = "Starting external node"; diff --git a/zk_toolbox/crates/zk_inception/src/server.rs b/zk_toolbox/crates/zk_inception/src/server.rs index 6773d224cba3..c4feb1c7c272 100644 --- a/zk_toolbox/crates/zk_inception/src/server.rs +++ b/zk_toolbox/crates/zk_inception/src/server.rs @@ -44,14 +44,18 @@ impl RunServer { } } - pub fn run(&self, shell: &Shell, server_mode: ServerMode) -> anyhow::Result<()> { + pub fn run( + &self, + shell: &Shell, + server_mode: ServerMode, + mut additional_args: Vec, + ) -> anyhow::Result<()> { shell.change_dir(&self.code_path); let config_genesis = &self.genesis.to_str().unwrap(); let config_wallets = &self.wallets.to_str().unwrap(); let config_general_config = &self.general_config.to_str().unwrap(); let config_contracts = &self.contracts.to_str().unwrap(); let secrets = &self.secrets.to_str().unwrap(); - let mut additional_args = vec![]; if let Some(components) = self.components() { additional_args.push(format!("--components={}", components)) } diff --git a/zk_toolbox/crates/zk_inception/src/forge_utils.rs b/zk_toolbox/crates/zk_inception/src/utils/forge.rs similarity index 100% rename from zk_toolbox/crates/zk_inception/src/forge_utils.rs rename to zk_toolbox/crates/zk_inception/src/utils/forge.rs diff --git a/zk_toolbox/crates/zk_inception/src/utils/mod.rs b/zk_toolbox/crates/zk_inception/src/utils/mod.rs new file mode 100644 index 000000000000..a84f0a336de5 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod forge; +pub mod rocks_db; diff --git a/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs new file mode 100644 index 000000000000..fc80aca100bc --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs @@ -0,0 +1,39 @@ +use std::path::Path; + +use config::RocksDbs; +use xshell::Shell; + +use crate::defaults::{ + EN_ROCKS_DB_PREFIX, MAIN_ROCKS_DB_PREFIX, ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE, +}; + +pub enum RocksDBDirOption { + Main, + ExternalNode, +} + +impl RocksDBDirOption { + pub fn prefix(&self) -> &str { + match self { + RocksDBDirOption::Main => MAIN_ROCKS_DB_PREFIX, + RocksDBDirOption::ExternalNode => EN_ROCKS_DB_PREFIX, + } + } +} + +pub fn recreate_rocksdb_dirs( + shell: &Shell, + rocks_db_path: &Path, + option: RocksDBDirOption, +) -> anyhow::Result { + let state_keeper = rocks_db_path + .join(option.prefix()) + .join(ROCKS_DB_STATE_KEEPER); + shell.remove_path(&state_keeper)?; + let merkle_tree = rocks_db_path.join(option.prefix()).join(ROCKS_DB_TREE); + shell.remove_path(&merkle_tree)?; + Ok(RocksDbs { + state_keeper: shell.create_dir(state_keeper)?, + merkle_tree: shell.create_dir(merkle_tree)?, + }) +} diff --git a/zk_toolbox/crates/zk_supervisor/Cargo.toml b/zk_toolbox/crates/zk_supervisor/Cargo.toml index 79d2bac74905..d8f5d7862a04 100644 --- a/zk_toolbox/crates/zk_supervisor/Cargo.toml +++ b/zk_toolbox/crates/zk_supervisor/Cargo.toml @@ -21,3 +21,4 @@ strum_macros.workspace = true tokio.workspace = true url.workspace = true xshell.workspace = true +serde.workspace = true diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs b/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs index c5b1229dd2ce..c506f7d07894 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs @@ -1,30 +1,54 @@ -use common::{cmd::Cmd, logger, spinner::Spinner}; +use clap::Parser; +use common::{cmd::Cmd, config::global_config, logger, spinner::Spinner}; use config::EcosystemConfig; +use serde::{Deserialize, Serialize}; use xshell::{cmd, Shell}; use crate::messages::{ - MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS, MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES, - MSG_INTEGRATION_TESTS_RUN_INFO, MSG_INTEGRATION_TESTS_RUN_SUCCESS, + msg_integration_tests_run, MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS, + MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES, MSG_INTEGRATION_TESTS_RUN_SUCCESS, }; +#[derive(Debug, Serialize, Deserialize, Parser)] +pub struct IntegrationTestCommands { + #[clap(short, long)] + external_node: bool, +} + const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; const CONTRACTS_TEST_DATA_PATH: &str = "etc/contracts-test-data"; -pub fn run(shell: &Shell) -> anyhow::Result<()> { +pub fn run( + shell: &Shell, + integration_test_commands: IntegrationTestCommands, +) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); - logger::info(MSG_INTEGRATION_TESTS_RUN_INFO); + logger::info(msg_integration_tests_run( + integration_test_commands.external_node, + )); build_repository(shell, &ecosystem_config)?; build_test_contracts(shell, &ecosystem_config)?; - Cmd::new( - cmd!(shell, "yarn jest --forceExit --testTimeout 60000") - .env("CHAIN_NAME", ecosystem_config.default_chain), - ) - .with_force_run() - .run()?; + let mut command = cmd!(shell, "yarn jest --forceExit --testTimeout 60000") + .env("CHAIN_NAME", ecosystem_config.default_chain); + + if integration_test_commands.external_node { + command = command.env( + "EXTERNAL_NODE", + format!("{:?}", integration_test_commands.external_node), + ) + } + if global_config().verbose { + command = command.env( + "ZKSYNC_DEBUG_LOGS", + format!("{:?}", global_config().verbose), + ) + } + + Cmd::new(command).with_force_run().run()?; logger::outro(MSG_INTEGRATION_TESTS_RUN_SUCCESS); diff --git a/zk_toolbox/crates/zk_supervisor/src/dals.rs b/zk_toolbox/crates/zk_supervisor/src/dals.rs index f2f6f86cfc61..ae8815c96899 100644 --- a/zk_toolbox/crates/zk_supervisor/src/dals.rs +++ b/zk_toolbox/crates/zk_supervisor/src/dals.rs @@ -1,10 +1,10 @@ -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use common::config::global_config; use config::{EcosystemConfig, SecretsConfig}; use url::Url; use xshell::Shell; -use crate::messages::MSG_CHAIN_NOT_FOUND_ERR; +use crate::messages::{MSG_CHAIN_NOT_FOUND_ERR, MSG_PROVER_URL_MUST_BE_PRESENTED}; const CORE_DAL_PATH: &str = "core/lib/dal"; const PROVER_DAL_PATH: &str = "prover/prover_dal"; @@ -46,7 +46,11 @@ pub fn get_prover_dal(shell: &Shell) -> anyhow::Result { Ok(Dal { path: PROVER_DAL_PATH.to_string(), - url: secrets.database.prover_url.clone(), + url: secrets + .database + .prover_url + .context(MSG_PROVER_URL_MUST_BE_PRESENTED)? + .clone(), }) } diff --git a/zk_toolbox/crates/zk_supervisor/src/main.rs b/zk_toolbox/crates/zk_supervisor/src/main.rs index ab5629465a88..96ab59bdad10 100644 --- a/zk_toolbox/crates/zk_supervisor/src/main.rs +++ b/zk_toolbox/crates/zk_supervisor/src/main.rs @@ -12,6 +12,8 @@ use messages::{ }; use xshell::Shell; +use crate::commands::integration_tests::IntegrationTestCommands; + mod commands; mod dals; mod messages; @@ -30,7 +32,7 @@ enum SupervisorSubcommands { #[command(subcommand, about = MSG_SUBCOMMAND_DATABASE_ABOUT)] Database(DatabaseCommands), #[command(about = MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT)] - IntegrationTests, + IntegrationTests(IntegrationTestCommands), } #[derive(Parser, Debug)] @@ -93,7 +95,9 @@ async fn main() -> anyhow::Result<()> { async fn run_subcommand(args: Supervisor, shell: &Shell) -> anyhow::Result<()> { match args.command { SupervisorSubcommands::Database(command) => commands::database::run(shell, command).await?, - SupervisorSubcommands::IntegrationTests => commands::integration_tests::run(shell)?, + SupervisorSubcommands::IntegrationTests(args) => { + commands::integration_tests::run(shell, args)? + } } Ok(()) } diff --git a/zk_toolbox/crates/zk_supervisor/src/messages.rs b/zk_toolbox/crates/zk_supervisor/src/messages.rs index 31bdb0eb9b1d..7ef956b8f545 100644 --- a/zk_toolbox/crates/zk_supervisor/src/messages.rs +++ b/zk_toolbox/crates/zk_supervisor/src/messages.rs @@ -1,5 +1,6 @@ // Ecosystem related messages pub(super) const MSG_CHAIN_NOT_FOUND_ERR: &str = "Chain not found"; + pub(super) fn msg_global_chain_does_not_exist(chain: &str, available_chains: &str) -> String { format!("Chain with name {chain} doesnt exist, please choose one of: {available_chains}") } @@ -10,12 +11,15 @@ pub(super) const MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT: &str = "Run integration // Database related messages pub(super) const MSG_NO_DATABASES_SELECTED: &str = "No databases selected"; + pub(super) fn msg_database_info(gerund_verb: &str) -> String { format!("{gerund_verb} databases") } + pub(super) fn msg_database_success(past_verb: &str) -> String { format!("Databases {past_verb} successfully") } + pub(super) fn msg_database_loading(gerund_verb: &str, dal: &str) -> String { format!("{gerund_verb} database for dal {dal}...") } @@ -33,6 +37,8 @@ pub(super) const MSG_DATABASE_RESET_PAST: &str = "reset"; pub(super) const MSG_DATABASE_SETUP_GERUND: &str = "Setting up"; pub(super) const MSG_DATABASE_SETUP_PAST: &str = "set up"; +pub(super) const MSG_PROVER_URL_MUST_BE_PRESENTED: &str = "Prover url must be presented"; + pub(super) const MSG_DATABASE_COMMON_PROVER_HELP: &str = "Prover database"; pub(super) const MSG_DATABASE_COMMON_CORE_HELP: &str = "Core database"; pub(super) const MSG_DATABASE_NEW_MIGRATION_DATABASE_HELP: &str = @@ -57,13 +63,24 @@ pub(super) const MSG_DATABASE_NEW_MIGRATION_DB_PROMPT: &str = "What database do you want to create a new migration for?"; pub(super) const MSG_DATABASE_NEW_MIGRATION_NAME_PROMPT: &str = "How do you want to name the migration?"; + pub(super) fn msg_database_new_migration_loading(dal: &str) -> String { format!("Creating new database migration for dal {}...", dal) } + pub(super) const MSG_DATABASE_NEW_MIGRATION_SUCCESS: &str = "Migration created successfully"; // Integration tests related messages -pub(super) const MSG_INTEGRATION_TESTS_RUN_INFO: &str = "Running integration tests"; + +pub(super) fn msg_integration_tests_run(external_node: bool) -> String { + let base = "Running integration tests"; + if external_node { + format!("{} for external node", base) + } else { + format!("{} for main server", base) + } +} + pub(super) const MSG_INTEGRATION_TESTS_RUN_SUCCESS: &str = "Integration tests ran successfully"; pub(super) const MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES: &str = "Building repository dependencies..."; From ef752926691d768ea412d0fdc78f43a62f16cd15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Grze=C5=9Bkiewicz?= Date: Wed, 26 Jun 2024 15:06:53 +0200 Subject: [PATCH 4/6] fix(eth-sender): revert commit changing which type of txs we resend first (#2327) Signed-off-by: tomg10 --- core/node/eth_sender/src/eth_tx_manager.rs | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/core/node/eth_sender/src/eth_tx_manager.rs b/core/node/eth_sender/src/eth_tx_manager.rs index 44759728d7c4..a69c52651339 100644 --- a/core/node/eth_sender/src/eth_tx_manager.rs +++ b/core/node/eth_sender/src/eth_tx_manager.rs @@ -250,35 +250,49 @@ impl EthTxManager { .l1_interface .get_operator_nonce(l1_block_numbers) .await?; + + let non_blob_tx_to_resend = self + .apply_inflight_txs_statuses_and_get_first_to_resend( + storage, + l1_block_numbers, + operator_nonce, + None, + ) + .await?; + let blobs_operator_nonce = self .l1_interface .get_blobs_operator_nonce(l1_block_numbers) .await?; let blobs_operator_address = self.l1_interface.get_blobs_operator_account(); + let mut blob_tx_to_resend = None; if let Some(blobs_operator_nonce) = blobs_operator_nonce { // need to check if both nonce and address are `Some` if blobs_operator_address.is_none() { panic!("blobs_operator_address has to be set its nonce is known; qed"); } - if let Some(res) = self - .monitor_inflight_transactions_inner( + blob_tx_to_resend = self + .apply_inflight_txs_statuses_and_get_first_to_resend( storage, l1_block_numbers, blobs_operator_nonce, blobs_operator_address, ) - .await? - { - return Ok(Some(res)); - } + .await?; } - self.monitor_inflight_transactions_inner(storage, l1_block_numbers, operator_nonce, None) - .await + // We have to resend non-blob transactions first, otherwise in case of a temporary + // spike in activity, all Execute and PublishProof would need to wait until all commit txs + // are sent, which may take some time. We treat them as if they had higher priority. + if non_blob_tx_to_resend.is_some() { + Ok(non_blob_tx_to_resend) + } else { + Ok(blob_tx_to_resend) + } } - async fn monitor_inflight_transactions_inner( + async fn apply_inflight_txs_statuses_and_get_first_to_resend( &mut self, storage: &mut Connection<'_, Core>, l1_block_numbers: L1BlockNumbers, From 85386d314a934b7eaa0bf2707f6d5af039e93340 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 27 Jun 2024 12:19:36 +0300 Subject: [PATCH 5/6] fix(object-store): Consider some token source errors transient (#2331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Considers some token source GCS errors transient. ## Why ❔ Considering errors as transient leads to less abnormal application terminations. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. --- core/lib/object_store/src/gcs.rs | 36 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/core/lib/object_store/src/gcs.rs b/core/lib/object_store/src/gcs.rs index 2d4fae77ab80..fd883a53f3e2 100644 --- a/core/lib/object_store/src/gcs.rs +++ b/core/lib/object_store/src/gcs.rs @@ -107,21 +107,22 @@ fn is_transient_http_error(err: &reqwest::Error) -> bool { || err.status() == Some(StatusCode::SERVICE_UNAVAILABLE) } -fn has_transient_io_source(mut err: &(dyn StdError + 'static)) -> bool { +fn get_source<'a, T: StdError + 'static>(mut err: &'a (dyn StdError + 'static)) -> Option<&'a T> { loop { - if err.is::() { - // We treat any I/O errors as transient. This isn't always true, but frequently occurring I/O errors - // (e.g., "connection reset by peer") *are* transient, and treating an error as transient is a safer option, - // even if it can lead to unnecessary retries. - return true; + if let Some(err) = err.downcast_ref::() { + return Some(err); } - err = match err.source() { - Some(source) => source, - None => return false, - }; + err = err.source()?; } } +fn has_transient_io_source(err: &(dyn StdError + 'static)) -> bool { + // We treat any I/O errors as transient. This isn't always true, but frequently occurring I/O errors + // (e.g., "connection reset by peer") *are* transient, and treating an error as transient is a safer option, + // even if it can lead to unnecessary retries. + get_source::(err).is_some() +} + impl From for ObjectStoreError { fn from(err: HttpError) -> Self { let is_not_found = match &err { @@ -135,10 +136,17 @@ impl From for ObjectStoreError { if is_not_found { ObjectStoreError::KeyNotFound(err.into()) } else { - let is_transient = matches!( - &err, - HttpError::HttpClient(err) if is_transient_http_error(err) - ); + let is_transient = match &err { + HttpError::HttpClient(err) => is_transient_http_error(err), + HttpError::TokenSource(err) => { + // Token sources are mostly based on the `reqwest` HTTP client, so transient error detection + // can reuse the same logic. + let err = err.as_ref(); + has_transient_io_source(err) + || get_source::(err).is_some_and(is_transient_http_error) + } + HttpError::Response(_) => false, + }; ObjectStoreError::Other { is_transient, source: err.into(), From 9985c2659177656788a1f6143120eafccfccdae9 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Thu, 27 Jun 2024 13:21:57 +0400 Subject: [PATCH 6/6] feat(gas_adjuster): Use eth_feeHistory for both base fee and blobs (#2322) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Updated the codebase to support blob information in the `eth_feeHistory` RPC method. Changes GasAdjuster so that it only uses this method to retrieve info. ## Why ❔ Use dedicated RPC method for getting info instead of custom implementation. Less requests to L1. Less code to maintain. ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. --- core/lib/basic_types/src/web3/mod.rs | 16 +- core/lib/eth_client/src/clients/http/query.rs | 39 ++++- core/lib/eth_client/src/clients/mock.rs | 76 +++++----- core/lib/eth_client/src/lib.rs | 9 +- .../api_server/src/web3/namespaces/eth.rs | 6 + core/node/eth_sender/src/tests.rs | 21 ++- .../src/l1_gas_price/gas_adjuster/mod.rs | 126 +++++----------- .../src/l1_gas_price/gas_adjuster/tests.rs | 139 +++++++++--------- core/node/state_keeper/src/io/tests/tester.rs | 14 +- 9 files changed, 232 insertions(+), 214 deletions(-) diff --git a/core/lib/basic_types/src/web3/mod.rs b/core/lib/basic_types/src/web3/mod.rs index af9cd1eea3fc..cfeeaa533b36 100644 --- a/core/lib/basic_types/src/web3/mod.rs +++ b/core/lib/basic_types/src/web3/mod.rs @@ -827,6 +827,7 @@ pub enum TransactionCondition { } // `FeeHistory`: from `web3::types::fee_history` +// Adapted to support blobs. /// The fee history type returned from `eth_feeHistory` call. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] @@ -834,14 +835,25 @@ pub enum TransactionCondition { pub struct FeeHistory { /// Lowest number block of the returned range. pub oldest_block: BlockNumber, - /// A vector of block base fees per gas. This includes the next block after the newest of the returned range, because this value can be derived from the newest block. Zeroes are returned for pre-EIP-1559 blocks. + /// A vector of block base fees per gas. This includes the next block after the newest of the returned range, + /// because this value can be derived from the newest block. Zeroes are returned for pre-EIP-1559 blocks. #[serde(default)] // some node implementations skip empty lists pub base_fee_per_gas: Vec, /// A vector of block gas used ratios. These are calculated as the ratio of gas used and gas limit. #[serde(default)] // some node implementations skip empty lists pub gas_used_ratio: Vec, - /// A vector of effective priority fee per gas data points from a single block. All zeroes are returned if the block is empty. Returned only if requested. + /// A vector of effective priority fee per gas data points from a single block. All zeroes are returned if + /// the block is empty. Returned only if requested. pub reward: Option>>, + /// An array of base fees per blob gas for blocks. This includes the next block following the newest in the + /// returned range, as this value can be derived from the latest block. For blocks before EIP-4844, zeroes + /// are returned. + #[serde(default)] // some node implementations skip empty lists + pub base_fee_per_blob_gas: Vec, + /// An array showing the ratios of blob gas used in blocks. These ratios are calculated by dividing blobGasUsed + /// by the maximum blob gas per block. + #[serde(default)] // some node implementations skip empty lists + pub blob_gas_used_ratio: Vec, } // `SyncInfo`, `SyncState`: from `web3::types::sync_state` diff --git a/core/lib/eth_client/src/clients/http/query.rs b/core/lib/eth_client/src/clients/http/query.rs index 33d9838dc735..1dee9fb0fda5 100644 --- a/core/lib/eth_client/src/clients/http/query.rs +++ b/core/lib/eth_client/src/clients/http/query.rs @@ -8,7 +8,7 @@ use zksync_web3_decl::error::{ClientRpcContext, EnrichedClientError, EnrichedCli use super::{decl::L1EthNamespaceClient, Method, COUNTERS, LATENCIES}; use crate::{ types::{ExecutedTxStatus, FailureInfo}, - EthInterface, RawTransactionBytes, + BaseFees, EthInterface, RawTransactionBytes, }; #[async_trait] @@ -78,7 +78,15 @@ where &self, upto_block: usize, block_count: usize, - ) -> EnrichedClientResult> { + ) -> EnrichedClientResult> { + // Non-panicking conversion to u64. + fn cast_to_u64(value: U256, tag: &str) -> EnrichedClientResult { + u64::try_from(value).map_err(|_| { + let err = ClientError::Custom(format!("{tag} value does not fit in u64")); + EnrichedClientError::new(err, "cast_to_u64").with_arg("value", &value) + }) + } + const MAX_REQUEST_CHUNK: usize = 1024; COUNTERS.call[&(Method::BaseFeeHistory, self.component())].inc(); @@ -103,11 +111,34 @@ where .with_arg("chunk_size", &chunk_size) .with_arg("block", &chunk_end) .await?; - history.extend(fee_history.base_fee_per_gas); + + // Check that the lengths are the same. + // Per specification, the values should always be provided, and must be 0 for blocks + // prior to EIP-4844. + // https://ethereum.github.io/execution-apis/api-documentation/ + if fee_history.base_fee_per_gas.len() != fee_history.base_fee_per_blob_gas.len() { + tracing::error!( + "base_fee_per_gas and base_fee_per_blob_gas have different lengths: {} and {}", + fee_history.base_fee_per_gas.len(), + fee_history.base_fee_per_blob_gas.len() + ); + } + + for (base, blob) in fee_history + .base_fee_per_gas + .into_iter() + .zip(fee_history.base_fee_per_blob_gas) + { + let fees = BaseFees { + base_fee_per_gas: cast_to_u64(base, "base_fee_per_gas")?, + base_fee_per_blob_gas: blob, + }; + history.push(fees) + } } latency.observe(); - Ok(history.into_iter().map(|fee| fee.as_u64()).collect()) + Ok(history) } async fn get_pending_block_base_fee_per_gas(&self) -> EnrichedClientResult { diff --git a/core/lib/eth_client/src/clients/mock.rs b/core/lib/eth_client/src/clients/mock.rs index 03162c2cfeb4..9fbc5ceb4b2e 100644 --- a/core/lib/eth_client/src/clients/mock.rs +++ b/core/lib/eth_client/src/clients/mock.rs @@ -14,7 +14,7 @@ use zksync_web3_decl::client::{DynClient, MockClient, L1}; use crate::{ types::{ContractCallError, SignedCallResult, SigningError}, - BoundEthInterface, Options, RawTransactionBytes, + BaseFees, BoundEthInterface, Options, RawTransactionBytes, }; #[derive(Debug, Clone)] @@ -212,8 +212,7 @@ type CallHandler = pub struct MockEthereumBuilder { max_fee_per_gas: U256, max_priority_fee_per_gas: U256, - base_fee_history: Vec, - excess_blob_gas_history: Vec, + base_fee_history: Vec, /// If true, the mock will not check the ordering nonces of the transactions. /// This is useful for testing the cases when the transactions are executed out of order. non_ordering_confirmations: bool, @@ -228,7 +227,6 @@ impl fmt::Debug for MockEthereumBuilder { .field("max_fee_per_gas", &self.max_fee_per_gas) .field("max_priority_fee_per_gas", &self.max_priority_fee_per_gas) .field("base_fee_history", &self.base_fee_history) - .field("excess_blob_gas_history", &self.excess_blob_gas_history) .field( "non_ordering_confirmations", &self.non_ordering_confirmations, @@ -244,7 +242,6 @@ impl Default for MockEthereumBuilder { max_fee_per_gas: 100.into(), max_priority_fee_per_gas: 10.into(), base_fee_history: vec![], - excess_blob_gas_history: vec![], non_ordering_confirmations: false, inner: Arc::default(), call_handler: Box::new(|call, block_id| { @@ -256,21 +253,13 @@ impl Default for MockEthereumBuilder { impl MockEthereumBuilder { /// Sets fee history for each block in the mocked Ethereum network, starting from the 0th block. - pub fn with_fee_history(self, history: Vec) -> Self { + pub fn with_fee_history(self, history: Vec) -> Self { Self { base_fee_history: history, ..self } } - /// Sets the excess blob gas history for each block in the mocked Ethereum network, starting from the 0th block. - pub fn with_excess_blob_gas_history(self, history: Vec) -> Self { - Self { - excess_blob_gas_history: history, - ..self - } - } - pub fn with_non_ordering_confirmation(self, non_ordering_confirmations: bool) -> Self { Self { non_ordering_confirmations, @@ -306,19 +295,16 @@ impl MockEthereumBuilder { } fn get_block_by_number( - base_fee_history: &[u64], - excess_blob_gas_history: &[u64], + fee_history: &[BaseFees], block: web3::BlockNumber, ) -> Option> { let web3::BlockNumber::Number(number) = block else { panic!("Non-numeric block requested"); }; - let excess_blob_gas = excess_blob_gas_history - .get(number.as_usize()) - .map(|excess_blob_gas| (*excess_blob_gas).into()); - let base_fee_per_gas = base_fee_history + let excess_blob_gas = Some(0.into()); // Not relevant for tests. + let base_fee_per_gas = fee_history .get(number.as_usize()) - .map(|base_fee| (*base_fee).into()); + .map(|fees| fees.base_fee_per_gas.into()); Some(web3::Block { number: Some(number), @@ -341,18 +327,12 @@ impl MockEthereumBuilder { move || Ok(U64::from(inner.read().unwrap().block_number)) }) .method("eth_getBlockByNumber", { - let base_fee_history = self.base_fee_history; - let excess_blob_gas_history = self.excess_blob_gas_history; move |number, full_transactions: bool| { assert!( !full_transactions, "getting blocks with transactions is not mocked" ); - Ok(Self::get_block_by_number( - &base_fee_history, - &excess_blob_gas_history, - number, - )) + Ok(Self::get_block_by_number(&self.base_fee_history, number)) } }) .method("eth_getTransactionCount", { @@ -374,10 +354,14 @@ impl MockEthereumBuilder { oldest_block: start_block.into(), base_fee_per_gas: base_fee_history[start_block..=from_block] .iter() - .copied() - .map(U256::from) + .map(|fee| U256::from(fee.base_fee_per_gas)) .collect(), - gas_used_ratio: vec![], // not used + base_fee_per_blob_gas: base_fee_history[start_block..=from_block] + .iter() + .map(|fee| fee.base_fee_per_blob_gas) + .collect(), + gas_used_ratio: vec![], // not used + blob_gas_used_ratio: vec![], // not used reward: None, }) }, @@ -591,10 +575,23 @@ mod tests { use super::*; use crate::{CallFunctionArgs, EthInterface}; + fn base_fees(block: u64, blob: u64) -> BaseFees { + BaseFees { + base_fee_per_gas: block, + base_fee_per_blob_gas: U256::from(blob), + } + } + #[tokio::test] async fn managing_block_number() { let mock = MockEthereum::builder() - .with_fee_history(vec![0, 1, 2, 3, 4]) + .with_fee_history(vec![ + base_fees(0, 4), + base_fees(1, 3), + base_fees(2, 2), + base_fees(3, 1), + base_fees(4, 0), + ]) .build(); let block_number = mock.client.block_number().await.unwrap(); assert_eq!(block_number, 0.into()); @@ -625,17 +622,24 @@ mod tests { #[tokio::test] async fn managing_fee_history() { + let initial_fee_history = vec![ + base_fees(1, 4), + base_fees(2, 3), + base_fees(3, 2), + base_fees(4, 1), + base_fees(5, 0), + ]; let client = MockEthereum::builder() - .with_fee_history(vec![1, 2, 3, 4, 5]) + .with_fee_history(initial_fee_history.clone()) .build(); client.advance_block_number(4); let fee_history = client.as_ref().base_fee_history(4, 4).await.unwrap(); - assert_eq!(fee_history, [2, 3, 4, 5]); + assert_eq!(fee_history, &initial_fee_history[1..=4]); let fee_history = client.as_ref().base_fee_history(2, 2).await.unwrap(); - assert_eq!(fee_history, [2, 3]); + assert_eq!(fee_history, &initial_fee_history[1..=2]); let fee_history = client.as_ref().base_fee_history(3, 2).await.unwrap(); - assert_eq!(fee_history, [3, 4]); + assert_eq!(fee_history, &initial_fee_history[2..=3]); } #[tokio::test] diff --git a/core/lib/eth_client/src/lib.rs b/core/lib/eth_client/src/lib.rs index 6e24047dd48c..b6ac3a89b54f 100644 --- a/core/lib/eth_client/src/lib.rs +++ b/core/lib/eth_client/src/lib.rs @@ -65,6 +65,13 @@ impl Options { } } +/// Information about the base fees provided by the L1 client. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct BaseFees { + pub base_fee_per_gas: u64, + pub base_fee_per_blob_gas: U256, +} + /// Common Web3 interface, as seen by the core applications. /// Encapsulates the raw Web3 interaction, providing a high-level interface. Acts as an extension /// trait implemented for L1 / Ethereum [clients](zksync_web3_decl::client::Client). @@ -96,7 +103,7 @@ pub trait EthInterface: Sync + Send { &self, from_block: usize, block_count: usize, - ) -> EnrichedClientResult>; + ) -> EnrichedClientResult>; /// Returns the `base_fee_per_gas` value for the currently pending L1 block. async fn get_pending_block_base_fee_per_gas(&self) -> EnrichedClientResult; diff --git a/core/node/api_server/src/web3/namespaces/eth.rs b/core/node/api_server/src/web3/namespaces/eth.rs index 397ce77c050f..33dfa277dc1c 100644 --- a/core/node/api_server/src/web3/namespaces/eth.rs +++ b/core/node/api_server/src/web3/namespaces/eth.rs @@ -688,6 +688,10 @@ impl EthNamespace { base_fee_per_gas.len() ]); + // We do not support EIP-4844, but per API specification we should return 0 for pre EIP-4844 blocks. + let base_fee_per_blob_gas = vec![U256::zero(); base_fee_per_gas.len()]; + let blob_gas_used_ratio = vec![0.0; base_fee_per_gas.len()]; + // `base_fee_per_gas` for next L2 block cannot be calculated, appending last fee as a placeholder. base_fee_per_gas.push(*base_fee_per_gas.last().unwrap()); Ok(FeeHistory { @@ -695,6 +699,8 @@ impl EthNamespace { base_fee_per_gas, gas_used_ratio, reward, + base_fee_per_blob_gas, + blob_gas_used_ratio, }) } diff --git a/core/node/eth_sender/src/tests.rs b/core/node/eth_sender/src/tests.rs index a3bb9951f44a..4853c7bb2299 100644 --- a/core/node/eth_sender/src/tests.rs +++ b/core/node/eth_sender/src/tests.rs @@ -9,7 +9,7 @@ use zksync_config::{ }; use zksync_contracts::BaseSystemContractsHashes; use zksync_dal::{Connection, ConnectionPool, Core, CoreDal}; -use zksync_eth_client::clients::MockEthereum; +use zksync_eth_client::{clients::MockEthereum, BaseFees}; use zksync_l1_contract_interface::i_executor::methods::{ExecuteBatches, ProveBatches}; use zksync_node_fee_model::l1_gas_price::GasAdjuster; use zksync_node_test_utils::{create_l1_batch, l1_batch_metadata_to_commitment_artifacts}; @@ -130,12 +130,23 @@ impl EthSenderTester { ..eth_sender_config.clone().sender.unwrap() }; + let history: Vec<_> = history + .into_iter() + .map(|base_fee_per_gas| BaseFees { + base_fee_per_gas, + base_fee_per_blob_gas: 0.into(), + }) + .collect(); + let gateway = MockEthereum::builder() .with_fee_history( - std::iter::repeat(0) - .take(Self::WAIT_CONFIRMATIONS as usize) - .chain(history) - .collect(), + std::iter::repeat_with(|| BaseFees { + base_fee_per_gas: 0, + base_fee_per_blob_gas: 0.into(), + }) + .take(Self::WAIT_CONFIRMATIONS as usize) + .chain(history) + .collect(), ) .with_non_ordering_confirmation(non_ordering_confirmations) .with_call_handler(move |call, _| { diff --git a/core/node/fee_model/src/l1_gas_price/gas_adjuster/mod.rs b/core/node/fee_model/src/l1_gas_price/gas_adjuster/mod.rs index 34cbee9b09e5..a3a1ed78e5b7 100644 --- a/core/node/fee_model/src/l1_gas_price/gas_adjuster/mod.rs +++ b/core/node/fee_model/src/l1_gas_price/gas_adjuster/mod.rs @@ -2,14 +2,13 @@ use std::{ collections::VecDeque, - ops::RangeInclusive, sync::{Arc, RwLock}, }; use tokio::sync::watch; use zksync_config::{configs::eth_sender::PubdataSendingMode, GasAdjusterConfig}; use zksync_eth_client::EthInterface; -use zksync_types::{commitment::L1BatchCommitmentMode, L1_GAS_PER_PUBDATA_BYTE, U256, U64}; +use zksync_types::{commitment::L1BatchCommitmentMode, L1_GAS_PER_PUBDATA_BYTE, U256}; use zksync_web3_decl::client::{DynClient, L1}; use self::metrics::METRICS; @@ -52,26 +51,25 @@ impl GasAdjuster { .await? .as_usize() .saturating_sub(1); - let base_fee_history = eth_client + let fee_history = eth_client .base_fee_history(current_block, config.max_base_fee_samples) .await?; - // Web3 API doesn't provide a method to fetch blob fees for multiple blocks using single request, - // so we request blob base fee only for the latest block. - let (_, last_block_blob_base_fee) = - Self::get_base_fees_history(eth_client.as_ref(), current_block..=current_block).await?; + let base_fee_statistics = GasStatistics::new( + config.max_base_fee_samples, + current_block, + fee_history.iter().map(|fee| fee.base_fee_per_gas), + ); + + let blob_base_fee_statistics = GasStatistics::new( + config.num_samples_for_blob_base_fee_estimate, + current_block, + fee_history.iter().map(|fee| fee.base_fee_per_blob_gas), + ); Ok(Self { - base_fee_statistics: GasStatistics::new( - config.max_base_fee_samples, - current_block, - &base_fee_history, - ), - blob_base_fee_statistics: GasStatistics::new( - config.num_samples_for_blob_base_fee_estimate, - current_block, - &last_block_blob_base_fee, - ), + base_fee_statistics, + blob_base_fee_statistics, config, pubdata_sending_mode, eth_client, @@ -95,25 +93,29 @@ impl GasAdjuster { let last_processed_block = self.base_fee_statistics.last_processed_block(); if current_block > last_processed_block { - let (base_fee_history, blob_base_fee_history) = Self::get_base_fees_history( - self.eth_client.as_ref(), - (last_processed_block + 1)..=current_block, - ) - .await?; + let n_blocks = current_block - last_processed_block; + let base_fees = self + .eth_client + .base_fee_history(current_block, n_blocks) + .await?; // We shouldn't rely on L1 provider to return consistent results, so we check that we have at least one new sample. - if let Some(current_base_fee_per_gas) = base_fee_history.last() { + if let Some(current_base_fee_per_gas) = base_fees.last().map(|fee| fee.base_fee_per_gas) + { METRICS .current_base_fee_per_gas - .set(*current_base_fee_per_gas); + .set(current_base_fee_per_gas); } - self.base_fee_statistics.add_samples(&base_fee_history); + self.base_fee_statistics + .add_samples(base_fees.iter().map(|fee| fee.base_fee_per_gas)); - if let Some(current_blob_base_fee) = blob_base_fee_history.last() { + if let Some(current_blob_base_fee) = + base_fees.last().map(|fee| fee.base_fee_per_blob_gas) + { // Blob base fee overflows `u64` only in very extreme cases. // It doesn't worth to observe exact value with metric because anyway values that can be used // are capped by `self.config.max_blob_base_fee()` of `u64` type. - if current_blob_base_fee > &U256::from(u64::MAX) { + if current_blob_base_fee > U256::from(u64::MAX) { tracing::error!("Failed to report current_blob_base_fee = {current_blob_base_fee}, it exceeds u64::MAX"); } else { METRICS @@ -122,7 +124,7 @@ impl GasAdjuster { } } self.blob_base_fee_statistics - .add_samples(&blob_base_fee_history); + .add_samples(base_fees.iter().map(|fee| fee.base_fee_per_blob_gas)); } Ok(()) } @@ -223,62 +225,6 @@ impl GasAdjuster { } } } - - /// Returns vector of base fees and blob base fees for given block range. - /// Note, that data for pre-dencun blocks won't be included in the vector returned. - async fn get_base_fees_history( - eth_client: &DynClient, - block_range: RangeInclusive, - ) -> anyhow::Result<(Vec, Vec)> { - let mut base_fee_history = Vec::new(); - let mut blob_base_fee_history = Vec::new(); - for block_number in block_range { - let header = eth_client.block(U64::from(block_number).into()).await?; - if let Some(base_fee_per_gas) = - header.as_ref().and_then(|header| header.base_fee_per_gas) - { - base_fee_history.push(base_fee_per_gas.as_u64()) - } - - if let Some(excess_blob_gas) = header.as_ref().and_then(|header| header.excess_blob_gas) - { - blob_base_fee_history.push(Self::blob_base_fee(excess_blob_gas.as_u64())) - } - } - - Ok((base_fee_history, blob_base_fee_history)) - } - - /// Calculates `blob_base_fee` given `excess_blob_gas`. - fn blob_base_fee(excess_blob_gas: u64) -> U256 { - // Constants and formula are taken from EIP4844 specification. - const MIN_BLOB_BASE_FEE: u32 = 1; - const BLOB_BASE_FEE_UPDATE_FRACTION: u32 = 3338477; - - Self::fake_exponential( - MIN_BLOB_BASE_FEE.into(), - excess_blob_gas.into(), - BLOB_BASE_FEE_UPDATE_FRACTION.into(), - ) - } - - /// approximates `factor * e ** (numerator / denominator)` using Taylor expansion. - fn fake_exponential(factor: U256, numerator: U256, denominator: U256) -> U256 { - let mut i = 1_u32; - let mut output = U256::zero(); - let mut accum = factor * denominator; - while !accum.is_zero() { - output += accum; - - accum *= numerator; - accum /= denominator; - accum /= U256::from(i); - - i += 1; - } - - output / denominator - } } impl L1TxParamsProvider for GasAdjuster { @@ -363,7 +309,7 @@ pub(super) struct GasStatisticsInner { } impl GasStatisticsInner { - fn new(max_samples: usize, block: usize, fee_history: &[T]) -> Self { + fn new(max_samples: usize, block: usize, fee_history: impl IntoIterator) -> Self { let mut statistics = Self { max_samples, samples: VecDeque::with_capacity(max_samples), @@ -387,9 +333,11 @@ impl GasStatisticsInner { self.samples.back().copied().unwrap_or(self.median_cached) } - fn add_samples(&mut self, fees: &[T]) { + fn add_samples(&mut self, fees: impl IntoIterator) { + let old_len = self.samples.len(); self.samples.extend(fees); - self.last_processed_block += fees.len(); + let processed_blocks = self.samples.len() - old_len; + self.last_processed_block += processed_blocks; let extra = self.samples.len().saturating_sub(self.max_samples); self.samples.drain(..extra); @@ -407,7 +355,7 @@ impl GasStatisticsInner { pub(super) struct GasStatistics(RwLock>); impl GasStatistics { - pub fn new(max_samples: usize, block: usize, fee_history: &[T]) -> Self { + pub fn new(max_samples: usize, block: usize, fee_history: impl IntoIterator) -> Self { Self(RwLock::new(GasStatisticsInner::new( max_samples, block, @@ -423,7 +371,7 @@ impl GasStatistics { self.0.read().unwrap().last_added_value() } - pub fn add_samples(&self, fees: &[T]) { + pub fn add_samples(&self, fees: impl IntoIterator) { self.0.write().unwrap().add_samples(fees) } diff --git a/core/node/fee_model/src/l1_gas_price/gas_adjuster/tests.rs b/core/node/fee_model/src/l1_gas_price/gas_adjuster/tests.rs index 594efc6915e2..200903b6deda 100644 --- a/core/node/fee_model/src/l1_gas_price/gas_adjuster/tests.rs +++ b/core/node/fee_model/src/l1_gas_price/gas_adjuster/tests.rs @@ -1,29 +1,29 @@ -use std::collections::VecDeque; +use std::{collections::VecDeque, sync::RwLockReadGuard}; use test_casing::test_casing; use zksync_config::{configs::eth_sender::PubdataSendingMode, GasAdjusterConfig}; -use zksync_eth_client::clients::MockEthereum; +use zksync_eth_client::{clients::MockEthereum, BaseFees}; use zksync_types::commitment::L1BatchCommitmentMode; -use super::{GasAdjuster, GasStatisticsInner}; +use super::{GasAdjuster, GasStatistics, GasStatisticsInner}; /// Check that we compute the median correctly #[test] fn median() { // sorted: 4 4 6 7 8 - assert_eq!(GasStatisticsInner::new(5, 5, &[6, 4, 7, 8, 4]).median(), 6); + assert_eq!(GasStatisticsInner::new(5, 5, [6, 4, 7, 8, 4]).median(), 6); // sorted: 4 4 8 10 - assert_eq!(GasStatisticsInner::new(4, 4, &[8, 4, 4, 10]).median(), 8); + assert_eq!(GasStatisticsInner::new(4, 4, [8, 4, 4, 10]).median(), 8); } /// Check that we properly manage the block base fee queue #[test] fn samples_queue() { - let mut stats = GasStatisticsInner::new(5, 5, &[6, 4, 7, 8, 4, 5]); + let mut stats = GasStatisticsInner::new(5, 5, [6, 4, 7, 8, 4, 5]); assert_eq!(stats.samples, VecDeque::from([4, 7, 8, 4, 5])); - stats.add_samples(&[18, 18, 18]); + stats.add_samples([18, 18, 18]); assert_eq!(stats.samples, VecDeque::from([4, 5, 18, 18, 18])); } @@ -32,38 +32,54 @@ fn samples_queue() { #[test_casing(2, [L1BatchCommitmentMode::Rollup, L1BatchCommitmentMode::Validium])] #[tokio::test] async fn kept_updated(commitment_mode: L1BatchCommitmentMode) { - let eth_client = MockEthereum::builder() - .with_fee_history(vec![0, 4, 6, 8, 7, 5, 5, 8, 10, 9]) - .with_excess_blob_gas_history(vec![ - 393216, - 393216 * 2, - 393216, - 393216 * 2, - 393216, - 393216 * 2, - 393216 * 3, - 393216 * 4, - ]) - .build(); + // Helper function to read a value from adjuster + fn read(statistics: &GasStatistics) -> RwLockReadGuard> { + statistics.0.read().unwrap() + } + + let block_fees = vec![0, 4, 6, 8, 7, 5, 5, 8, 10, 9]; + let blob_fees = vec![ + 0, + 393216, + 393216, + 393216 * 2, + 393216, + 393216 * 2, + 393216 * 2, + 393216 * 3, + 393216 * 4, + 393216, + ]; + let base_fees = block_fees + .into_iter() + .zip(blob_fees) + .map(|(block, blob)| BaseFees { + base_fee_per_gas: block, + base_fee_per_blob_gas: blob.into(), + }) + .collect(); + + let eth_client = MockEthereum::builder().with_fee_history(base_fees).build(); // 5 sampled blocks + additional block to account for latest block subtraction eth_client.advance_block_number(6); + let config = GasAdjusterConfig { + default_priority_fee_per_gas: 5, + max_base_fee_samples: 5, + pricing_formula_parameter_a: 1.5, + pricing_formula_parameter_b: 1.0005, + internal_l1_pricing_multiplier: 0.8, + internal_enforced_l1_gas_price: None, + internal_enforced_pubdata_price: None, + poll_period: 5, + max_l1_gas_price: None, + num_samples_for_blob_base_fee_estimate: 3, + internal_pubdata_pricing_multiplier: 1.0, + max_blob_base_fee: None, + }; let adjuster = GasAdjuster::new( Box::new(eth_client.clone().into_client()), - GasAdjusterConfig { - default_priority_fee_per_gas: 5, - max_base_fee_samples: 5, - pricing_formula_parameter_a: 1.5, - pricing_formula_parameter_b: 1.0005, - internal_l1_pricing_multiplier: 0.8, - internal_enforced_l1_gas_price: None, - internal_enforced_pubdata_price: None, - poll_period: 5, - max_l1_gas_price: None, - num_samples_for_blob_base_fee_estimate: 3, - internal_pubdata_pricing_multiplier: 1.0, - max_blob_base_fee: None, - }, + config, PubdataSendingMode::Calldata, commitment_mode, ) @@ -71,58 +87,35 @@ async fn kept_updated(commitment_mode: L1BatchCommitmentMode) { .unwrap(); assert_eq!( - adjuster.base_fee_statistics.0.read().unwrap().samples.len(), - 5 + read(&adjuster.base_fee_statistics).samples.len(), + config.max_base_fee_samples ); - assert_eq!(adjuster.base_fee_statistics.0.read().unwrap().median(), 6); + assert_eq!(read(&adjuster.base_fee_statistics).median(), 6); - let expected_median_blob_base_fee = GasAdjuster::blob_base_fee(393216); + eprintln!("{:?}", read(&adjuster.blob_base_fee_statistics).samples); + let expected_median_blob_base_fee = 393216 * 2; assert_eq!( - adjuster - .blob_base_fee_statistics - .0 - .read() - .unwrap() - .samples - .len(), - 1 + read(&adjuster.blob_base_fee_statistics).samples.len(), + config.num_samples_for_blob_base_fee_estimate ); assert_eq!( - adjuster.blob_base_fee_statistics.0.read().unwrap().median(), - expected_median_blob_base_fee + read(&adjuster.blob_base_fee_statistics).median(), + expected_median_blob_base_fee.into() ); eth_client.advance_block_number(3); adjuster.keep_updated().await.unwrap(); assert_eq!( - adjuster.base_fee_statistics.0.read().unwrap().samples.len(), - 5 + read(&adjuster.base_fee_statistics).samples.len(), + config.max_base_fee_samples ); - assert_eq!(adjuster.base_fee_statistics.0.read().unwrap().median(), 7); + assert_eq!(read(&adjuster.base_fee_statistics).median(), 7); - let expected_median_blob_base_fee = GasAdjuster::blob_base_fee(393216 * 3); + let expected_median_blob_base_fee = 393216 * 3; + assert_eq!(read(&adjuster.blob_base_fee_statistics).samples.len(), 3); assert_eq!( - adjuster - .blob_base_fee_statistics - .0 - .read() - .unwrap() - .samples - .len(), - 3 + read(&adjuster.blob_base_fee_statistics).median(), + expected_median_blob_base_fee.into() ); - assert_eq!( - adjuster.blob_base_fee_statistics.0.read().unwrap().median(), - expected_median_blob_base_fee - ); -} - -#[test] -fn blob_base_fee_formula() { - const EXCESS_BLOB_GAS: u64 = 0x4b80000; - const EXPECTED_BLOB_BASE_FEE: u64 = 19893400088; - - let blob_base_fee = GasAdjuster::blob_base_fee(EXCESS_BLOB_GAS); - assert_eq!(blob_base_fee.as_u64(), EXPECTED_BLOB_BASE_FEE); } diff --git a/core/node/state_keeper/src/io/tests/tester.rs b/core/node/state_keeper/src/io/tests/tester.rs index 35758c44bc95..f5a132baea3b 100644 --- a/core/node/state_keeper/src/io/tests/tester.rs +++ b/core/node/state_keeper/src/io/tests/tester.rs @@ -8,7 +8,7 @@ use zksync_config::{ }; use zksync_contracts::BaseSystemContracts; use zksync_dal::{ConnectionPool, Core, CoreDal}; -use zksync_eth_client::clients::MockEthereum; +use zksync_eth_client::{clients::MockEthereum, BaseFees}; use zksync_multivm::vm_latest::constants::BATCH_COMPUTATIONAL_GAS_LIMIT; use zksync_node_fee_model::{l1_gas_price::GasAdjuster, MainNodeFeeInputProvider}; use zksync_node_genesis::create_genesis_l1_batch; @@ -47,9 +47,15 @@ impl Tester { } async fn create_gas_adjuster(&self) -> GasAdjuster { - let eth_client = MockEthereum::builder() - .with_fee_history(vec![0, 4, 6, 8, 7, 5, 5, 8, 10, 9]) - .build(); + let block_fees = vec![0, 4, 6, 8, 7, 5, 5, 8, 10, 9]; + let base_fees = block_fees + .into_iter() + .map(|base_fee_per_gas| BaseFees { + base_fee_per_gas, + base_fee_per_blob_gas: 1.into(), // Not relevant for the test + }) + .collect(); + let eth_client = MockEthereum::builder().with_fee_history(base_fees).build(); let gas_adjuster_config = GasAdjusterConfig { default_priority_fee_per_gas: 10,