feat: Provide easy prover setup

prover/crates/bin/prover_cli/src/cli.rs

@@ -2,7 +2,8 @@ use clap::{command, Args, Parser, Subcommand};
 use zksync_types::url::SensitiveUrl;
 
 use crate::commands::{
-    config, debug_proof, delete, get_file_info, requeue, restart, stats, status::StatusCommand,
+    config, debug_proof, delete, get_file_info, insert_batch, insert_version, requeue, restart,
+    stats, status::StatusCommand,
 };
 
 pub const VERSION_STRING: &str = env!("CARGO_PKG_VERSION");
@@ -27,6 +28,8 @@ impl ProverCLI {
             ProverCommand::Restart(args) => restart::run(args).await?,
             ProverCommand::DebugProof(args) => debug_proof::run(args).await?,
             ProverCommand::Stats(args) => stats::run(args, self.config).await?,
+            ProverCommand::InsertVersion(args) => insert_version::run(args, self.config).await?,
+            ProverCommand::InsertBatch(args) => insert_batch::run(args, self.config).await?,
         };
         Ok(())
     }
@@ -55,4 +58,6 @@ pub enum ProverCommand {
     Restart(restart::Args),
     #[command(about = "Displays L1 Batch proving stats for a given period")]
     Stats(stats::Options),
+    InsertVersion(insert_version::Args),
+    InsertBatch(insert_batch::Args),
 }

prover/crates/bin/prover_cli/src/commands/insert_batch.rs

@@ -0,0 +1,43 @@
+use anyhow::Context as _;
+use clap::Args as ClapArgs;
+use zksync_basic_types::{
+    protocol_version::{ProtocolSemanticVersion, ProtocolVersionId, VersionPatch},
+    L1BatchNumber,
+};
+use zksync_db_connection::connection_pool::ConnectionPool;
+use zksync_prover_dal::{Prover, ProverDal};
+
+use crate::cli::ProverCLIConfig;
+
+#[derive(ClapArgs)]
+pub struct Args {
+    #[clap(short, long)]
+    pub number: L1BatchNumber,
+    #[clap(short, long)]
+    pub version: u16,
+    #[clap(short, long)]
+    pub patch: u32,
+}
+
+pub async fn run(args: Args, config: ProverCLIConfig) -> anyhow::Result<()> {
+    let connection = ConnectionPool::<Prover>::singleton(config.db_url)
+        .build()
+        .await
+        .context("failed to build a prover_connection_pool")?;
+    let mut conn = connection.connection().await.unwrap();
+
+    let protocol_version = ProtocolVersionId::try_from(args.version)
+        .map_err(|_| anyhow::anyhow!("Invalid protocol version"))?;
+
+    let protocol_version_patch = VersionPatch(args.patch);
+
+    conn.fri_witness_generator_dal()
+        .save_witness_inputs(
+            args.number,
+            &format!("witness_inputs_{}", args.number.0),
+            ProtocolSemanticVersion::new(protocol_version, protocol_version_patch),
+        )
+        .await;
+
+    Ok(())
+}

prover/crates/bin/prover_cli/src/commands/insert_version.rs

@@ -0,0 +1,52 @@
+use std::str::FromStr;
+
+use anyhow::Context as _;
+use clap::Args as ClapArgs;
+use zksync_basic_types::{
+    protocol_version::{
+        L1VerifierConfig, ProtocolSemanticVersion, ProtocolVersionId, VersionPatch,
+    },
+    H256,
+};
+use zksync_db_connection::connection_pool::ConnectionPool;
+use zksync_prover_dal::{Prover, ProverDal};
+
+use crate::cli::ProverCLIConfig;
+
+#[derive(ClapArgs)]
+pub struct Args {
+    #[clap(short, long)]
+    pub version: u16,
+    #[clap(short, long)]
+    pub patch: u32,
+    #[clap(short, long)]
+    pub snark_wrapper: String,
+}
+
+pub async fn run(args: Args, config: ProverCLIConfig) -> anyhow::Result<()> {
+    let connection = ConnectionPool::<Prover>::singleton(config.db_url)
+        .build()
+        .await
+        .context("failed to build a prover_connection_pool")?;
+    let mut conn = connection.connection().await.unwrap();
+
+    let protocol_version = ProtocolVersionId::try_from(args.version)
+        .map_err(|_| anyhow::anyhow!("Invalid protocol version"))?;
+
+    let protocol_version_patch = VersionPatch(args.patch);
+
+    let snark_wrapper = H256::from_str(&args.snark_wrapper).unwrap_or_else(|_| {
+        panic!("Invalid snark wrapper hash");
+    });
+
+    conn.fri_protocol_versions_dal()
+        .save_prover_protocol_version(
+            ProtocolSemanticVersion::new(protocol_version, protocol_version_patch),
+            L1VerifierConfig {
+                recursion_scheduler_level_vk_hash: snark_wrapper,
+            },
+        )
+        .await;
+
+    Ok(())
+}

prover/crates/bin/prover_cli/src/commands/mod.rs

@@ -2,6 +2,8 @@ pub(crate) mod config;
 pub(crate) mod debug_proof;
 pub(crate) mod delete;
 pub(crate) mod get_file_info;
+pub(crate) mod insert_batch;
+pub(crate) mod insert_version;
 pub(crate) mod requeue;
 pub(crate) mod restart;
 pub(crate) mod stats;

prover/crates/bin/witness_generator/README.md

@@ -50,76 +50,3 @@ One round of prover generation consists of:
 
 Note that the very first input table (`witness_inputs`) is populated by the tree (as the input artifact for the
 `WitnessGeneratorJobType::BasicCircuits` is the merkle proofs)
-
-## Running BWG for custom batch
-
-After releases `prover-v15.1.0` and `core-v24.9.0` basic witness generator doesn't need access to core database anymore.
-Database information now lives in input file, called `witness_inputs_<batch>.bin` generated by different core
-components).
-
-This file is stored by prover gateway in GCS (or your choice of object storage -- check config). To access it from GCS
-(assuming you have access to the bucket), run:
-
-```shell
-gsutil cp gs://your_bucket/witness_inputs/witness_inputs_<batch>.bin <path/to/era/prover/artifacts/witness_inputs>
-```
-
-Note, that you need to have `gsutil` installed, and you need to have access to the bucket.
-
-Now, database needs to know about the batch and the protocol version it should use. Check the latest protocol version in
-the codebase by checking const `PROVER_PROTOCOL_SEMANTIC_VERSION` or run the binary in `prover` workspace:
-
-```console
-cargo run --bin prover_version
-```
-
-It will give you the latest prover protocol version in a semver format, like `0.24.2`, you need to know only minor and
-patch versions. Now, go to the `prover/crates/bin/vk_setup_data_generator_server_fri/data/commitments.json` and get
-`snark_wrapper` value from it. Then, you need to insert the info about protocol version into the database. First,
-connect to the database, e.g. locally you can do it like that:
-
-```shell
-psql postgres://postgres:notsecurepassword@localhost/prover_local
-```
-
-And run the following query:
-
-```shell
-INSERT INTO
-prover_fri_protocol_versions (
-id,
-recursion_scheduler_level_vk_hash,
-created_at,
-protocol_version_patch
-)
-VALUES
-(<minor version>, '<snark wrapper value>'::bytea, NOW(), <patch version>)
-ON CONFLICT (id, protocol_version_patch) DO NOTHING
-
-```
-
-Now, you need to insert the batch into the database. Run the following query:
-
-```shell
-INSERT INTO
-witness_inputs_fri (
-l1_batch_number,
-witness_inputs_blob_url,
-protocol_version,
-status,
-created_at,
-updated_at,
-protocol_version_patch
-)
-VALUES
-(<batch number>, 'witness_inputs_<batch_number>.bin', <minor version>, 'queued', NOW(), NOW(), <patch version>)
-ON CONFLICT (l1_batch_number) DO NOTHING
-```
-
-Finally, run the basic witness generator itself:
-
-```shell
-API_PROMETHEUS_LISTENER_PORT=3116 zk f cargo run --release --bin zksync_witness_generator -- --round=basic_circuits
-```
-
-And you are good to go!

prover/docs/05_proving_batch.md

@@ -0,0 +1,110 @@
+# Proving a batch
+
+If you got to this section, then most likely you are wondering how to prove and verify the batch by yourself. After
+releases `prover-v15.1.0` and `core-v24.9.0` prover subsystem doesn't need access to core database anymore, which means
+you can run only prover subsystem and prove batches without running the whole core system. This guide will help you with
+that.
+
+## Requirements
+
+First of all, you need to install CUDA drivers, all other things will be dealt with by `zk_inception` and `prover_cli`
+tools. For that, check the following [guide](./02_setup.md)(you can skip bellman-cuda step).
+
+Now, you can use `zk_inception` tool for setting up the env and running prover subsystem. Check the steps for
+installation in [this guide](../../zk_toolbox/README.md). And don't forget to install the prerequisites!
+
+Also, to assure that you are working with database securely, you need to use `prover_cli` tool. You can find the guide
+for installation [here](../crates/bin/prover_cli/README.md).
+
+## Initializing system
+
+After you have installed the tool, you can run the prover subsystem by running:
+
+```shell
+zk_inception ecosystem create
+```
+
+The command will create the ecosystem and all the necessary components for the prover subsystem. Now, you need to
+initialize the prover subsystem by running:
+
+```shell
+zk_inception prover init
+```
+
+For the prompts, you need to put `Yes` for copying the configs and setting up the database.
+
+## Proving the batch
+
+### Getting data needed for proving
+
+At this step, we need to get the witness inputs data for the batch you want to prove. Database information now lives in
+input file, called `witness_inputs_<batch>.bin` generated by different core components).
+
+- If batch was produced by your system, the file is stored by prover gateway in GCS (or your choice of object storage --
+  check config). To access it from GCS (assuming you have access to the bucket), run:
+
+  ```shell
+  gsutil cp gs://your_bucket/witness_inputs/witness_inputs_<batch>.bin <path/to/era/prover/artifacts/witness_inputs>
+  ```
+
+- If you want to prove the batch produced by zkSync, you can get the data from the `ExternalProofIntegrationAPI` using
+  `{address}/proof_generation_data` endpoint. You need to replace `{address}` with the address of the API and provide
+  the batch number as a query data to get the data for specific batch, otherwise, you will receive latest data for the
+  batch, that was already proven.
+
+### Preparing database
+
+After you have the data, you need to prepare the system to run the batch. So, database needs to know about the batch and
+the protocol version it should use. Check the latest protocol version in the codebase by checking const
+`PROVER_PROTOCOL_SEMANTIC_VERSION` or run the binary in `prover` workspace:
+
+```console
+cargo run --bin prover_version
+```
+
+It will give you the latest prover protocol version in a semver format, like `0.24.2`, you need to know only minor and
+patch versions. Now, go to the `prover/crates/bin/vk_setup_data_generator_server_fri/data/commitments.json` and get
+`snark_wrapper` value from it. Then, you need to insert the info about protocol version into the database. First,
+connect to the database, e.g. locally you can do it like that:
+
+Now, with the use of `prover_cli` tool, you can insert the data about the batch and protocol version into the database:
+
+First, let the tool know which database it should connect to((for local DB you can find the url in `secrets.yaml` file):
+
+```shell
+pli config --db-url <DATABASE_URL>
+```
+
+Now, insert the information about protocol version in the database:
+
+```shell
+pli insert-version --version=<MINOR_VERSION> --patch=<PATCH_VERSION> --snark-wrapper=<SNARK_WRAPPER>
+```
+
+And finally, provide the data about the batch:
+
+```shell
+pli insert-batch --number=<BATCH_NUMBER> --version=<MINOR_VERSION> --patch=<PATCH_VERSION>
+```
+
+## Running prover subsystem
+
+At this step, all the data is prepared and you can run the prover subsystem. To do that, run the following commands:
+
+```shell
+zk_inception prover run --component=prover
+zk_inception prover run --component=witness-generator --round=all-rounds
+zk_inception prover run --component=witness-vector-generator --threads=10
+zk_inception prover run --component=compressor
+```
+
+And you are good to go! The prover subsystem will prove the batch and you can check the results in the database.
+
+## Verifying zkSync batch
+
+Now, assuming the proof is already generated, you can verify using `ExternalProofIntegrationAPI`. Usually proof is
+stored in GCS bucket(for which you can use the same steps as for getting the witness inputs data
+[here](#getting-data-needed-for-proving), but locally you can find it in `/artifacts/proofs_fri` directory). Now, simply
+send the data to the endpoint `{address}/verify_batch/{batch_number}`. Note, that you need to pass the generated proof
+as serialized JSON data when calling the endpoint. API will respond with status 200 if the proof is valid and with the
+error message otherwise.

zk_toolbox/crates/config/src/secrets.rs

@@ -26,6 +26,18 @@ pub fn set_databases(
     Ok(())
 }
 
+pub fn set_prover_database(
+    secrets: &mut SecretsConfig,
+    prover_db_config: &DatabaseConfig,
+) -> anyhow::Result<()> {
+    let database = secrets
+        .database
+        .as_mut()
+        .context("Databases must be presented")?;
+    database.prover_url = Some(SensitiveUrl::from(prover_db_config.full_url()));
+    Ok(())
+}
+
 pub fn set_l1_rpc_url(secrets: &mut SecretsConfig, l1_rpc_url: String) -> anyhow::Result<()> {
     secrets
         .l1

zk_toolbox/crates/zk_inception/README.md

@@ -269,6 +269,7 @@ Initialize chain, deploying necessary contracts and performing on-chain operatio
 - `-u`, `--use-default` — Use default database urls and names
 - `-d`, `--dont-drop`
 - `--deploy-paymaster <DEPLOY_PAYMASTER>`
+- `--copy-configs` - Copy default configs to the chain
 
   Possible values: `true`, `false`
 
@@ -418,7 +419,7 @@ Initialize prover
 - `--project-id <PROJECT_ID>`
 - `--shall-save-to-public-bucket <SHALL_SAVE_TO_PUBLIC_BUCKET>`
 
-  Possible values: `true`, `false`
+Possible values: `true`, `false`
 
 - `--public-store-dir <PUBLIC_STORE_DIR>`
 - `--public-bucket-base-url <PUBLIC_BUCKET_BASE_URL>`
@@ -428,8 +429,10 @@ Initialize prover
 - `--public-project-id <PUBLIC_PROJECT_ID>`
 - `--bellman-cuda-dir <BELLMAN_CUDA_DIR>`
 - `--download-key <DOWNLOAD_KEY>`
+- `--setup-database`
+- `--copy-configs`
 
-  Possible values: `true`, `false`
+Possible values: `true`, `false`
 
 - `--setup-key-path <SETUP_KEY_PATH>`
 - `--cloud-type <CLOUD_TYPE>`