Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(api): Make Web3 API server work with pruned data #838

Merged
Merged
Changes from 28 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
0dba582
Brush up `debug` namespace
slowli Jan 3, 2024
8123935
Sketch pruning checks in API server
slowli Jan 3, 2024
bf1b4db
Add basic tests for `debug` namespace
slowli Jan 4, 2024
695a6e2
Test `debug` endpoints with snapshot recovery
slowli Jan 4, 2024
0feb18f
Rework API server initialization
slowli Jan 4, 2024
41cce22
Make latest miniblock number optional in DAL
slowli Jan 5, 2024
1985632
Remove `BlocksWeb3Dal::get_sealed_l1_batch_number()`
slowli Jan 5, 2024
ebbf29d
Fix pending block resolution
slowli Jan 5, 2024
e9e4bda
Add TODO for propagating DB errors in `TxSender`
slowli Jan 5, 2024
ee99e9c
Check state pruning during `BlockArgs` initialization
slowli Jan 5, 2024
b25d751
Test basic methods after snapshot recovery
slowli Jan 5, 2024
3e84404
Fix getting starting miniblock number in notifiers
slowli Jan 8, 2024
504e87e
Test snapshot recovery with subscriptions
slowli Jan 8, 2024
6ed8cc0
Test snapshot recovery for filters
slowli Jan 8, 2024
f286151
Adapt L1 batch resolution for miniblocks
slowli Jan 8, 2024
a43fd8f
Test VM-related API methods
slowli Jan 8, 2024
0aab093
Update from upstream
slowli Jan 8, 2024
83221a1
Split off filter- and VM-related API tests
slowli Jan 9, 2024
cd5d8cc
Test `TxSender::get_expected_nonce()`
slowli Jan 9, 2024
0cfdbcb
Check L1 batch pruning in relevant `zks` methods
slowli Jan 9, 2024
91fd3ad
Adapt `next_nonce_by_initiator_account()` for snapshot recovery
slowli Jan 9, 2024
a5a53a3
Test `get_transaction_count()` method
slowli Jan 9, 2024
c1cca08
Fix resolution of `BlockNumber::Earliest`
slowli Jan 9, 2024
959ce08
Test `BlockArgs::new()`
slowli Jan 9, 2024
05a8c6e
Update from upstream
slowli Jan 9, 2024
fdc63da
Fix DB query for pending nonce
slowli Jan 10, 2024
377692f
Update from upstream
slowli Jan 16, 2024
3ced742
Update from upstream
slowli Jan 16, 2024
d382eb6
Revert reordering `derive`s
slowli Jan 17, 2024
b1bb73a
Refactor `BlockStartInfo`
slowli Jan 17, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion core/bin/storage_logs_dedup_migration/src/main.rs
slowli marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -57,7 +57,8 @@ async fn main() {
.blocks_dal()
.get_sealed_miniblock_number()
.await
.unwrap();
.unwrap()
.expect("Cannot start migration for Postgres recovered from snapshot");
println!(
"Migration started for miniblock range {}..={}",
opt.start_from_miniblock, sealed_miniblock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

This file was deleted.

This file was deleted.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 5 additions & 6 deletions core/lib/dal/src/blocks_dal.rs
Original file line number Diff line number Diff line change
@@ -61,8 +61,8 @@ impl BlocksDal<'_, '_> {
Ok(row.number.map(|num| L1BatchNumber(num as u32)))
}

pub async fn get_sealed_miniblock_number(&mut self) -> sqlx::Result<MiniblockNumber> {
let number: i64 = sqlx::query!(
pub async fn get_sealed_miniblock_number(&mut self) -> sqlx::Result<Option<MiniblockNumber>> {
let row = sqlx::query!(
r#"
SELECT
MAX(number) AS "number"
@@ -73,10 +73,9 @@ impl BlocksDal<'_, '_> {
.instrument("get_sealed_miniblock_number")
.report_latency()
.fetch_one(self.storage.conn())
.await?
.number
.unwrap_or(0);
Ok(MiniblockNumber(number as u32))
.await?;

Ok(row.number.map(|number| MiniblockNumber(number as u32)))
}

/// Returns the number of the earliest L1 batch present in the DB, or `None` if there are no L1 batches.
147 changes: 82 additions & 65 deletions core/lib/dal/src/blocks_web3_dal.rs
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ use crate::{
models::{
storage_block::{
bind_block_where_sql_params, web3_block_number_to_sql, web3_block_where_sql,
StorageBlockDetails, StorageL1BatchDetails,
ResolvedL1BatchForMiniblock, StorageBlockDetails, StorageL1BatchDetails,
},
storage_transaction::{extract_web3_transaction, web3_transaction_select_sql, CallTrace},
},
@@ -32,42 +32,6 @@ pub struct BlocksWeb3Dal<'a, 'c> {
}

impl BlocksWeb3Dal<'_, '_> {
pub async fn get_sealed_miniblock_number(&mut self) -> sqlx::Result<MiniblockNumber> {
let number = sqlx::query!(
r#"
SELECT
MAX(number) AS "number"
FROM
miniblocks
"#
)
.instrument("get_sealed_block_number")
.report_latency()
.fetch_one(self.storage.conn())
.await?
.number
.expect("DAL invocation before genesis");
Ok(MiniblockNumber(number as u32))
}

pub async fn get_sealed_l1_batch_number(&mut self) -> sqlx::Result<L1BatchNumber> {
let number = sqlx::query!(
r#"
SELECT
MAX(number) AS "number"
FROM
l1_batches
"#
)
.instrument("get_sealed_block_number")
.report_latency()
.fetch_one(self.storage.conn())
.await?
.number
.expect("DAL invocation before genesis");
Ok(L1BatchNumber(number as u32))
}

pub async fn get_block_by_web3_block_id(
&mut self,
block_id: api::BlockId,
@@ -258,21 +222,26 @@ impl BlocksWeb3Dal<'_, '_> {
&mut self,
block_id: api::BlockId,
) -> sqlx::Result<Option<MiniblockNumber>> {
let query_string = match block_id {
api::BlockId::Hash(_) => "SELECT number FROM miniblocks WHERE hash = $1".to_owned(),
let query_string;
let query_str = match block_id {
api::BlockId::Hash(_) => "SELECT number FROM miniblocks WHERE hash = $1",
api::BlockId::Number(api::BlockNumber::Number(_)) => {
// The reason why instead of returning the `block_number` directly we use query is
// to handle numbers of blocks that are not created yet.
// the `SELECT number FROM miniblocks WHERE number=block_number` for
// non-existing block number will returns zero.
"SELECT number FROM miniblocks WHERE number = $1".to_owned()
// to handle numbers of blocks that are not created yet or were pruned.
// The query below will return NULL for non-existing block numbers.
"SELECT number FROM miniblocks WHERE number = $1"
}
api::BlockId::Number(api::BlockNumber::Earliest) => {
return Ok(Some(MiniblockNumber(0)));
// Similarly to `BlockNumber::Number`, we may be missing the earliest block
// if the storage was recovered from a snapshot.
"SELECT number FROM miniblocks WHERE number = 0"
}
api::BlockId::Number(block_number) => {
query_string = web3_block_number_to_sql(block_number);
&query_string
}
api::BlockId::Number(block_number) => web3_block_number_to_sql(block_number),
};
let row = bind_block_where_sql_params(&block_id, sqlx::query(&query_string))
let row = bind_block_where_sql_params(&block_id, sqlx::query(query_str))
.fetch_optional(self.storage.conn())
.await?;

@@ -283,31 +252,33 @@ impl BlocksWeb3Dal<'_, '_> {
}

/// Returns L1 batch timestamp for either sealed or pending L1 batch.
///
/// The correctness of the current implementation depends on the timestamp of an L1 batch always
/// being equal to the timestamp of the first miniblock in the batch.
pub async fn get_expected_l1_batch_timestamp(
&mut self,
l1_batch_number: L1BatchNumber,
l1_batch_number: &ResolvedL1BatchForMiniblock,
) -> sqlx::Result<Option<u64>> {
let first_miniblock_of_batch = if l1_batch_number.0 == 0 {
MiniblockNumber(0)
} else {
match self
.get_miniblock_range_of_l1_batch(l1_batch_number - 1)
.await?
{
Some((_, miniblock_number)) => miniblock_number + 1,
None => return Ok(None),
}
};
let timestamp = sqlx::query!(
r#"
SELECT
timestamp
FROM
miniblocks
WHERE
number = $1
(
$1::BIGINT IS NULL
AND l1_batch_number IS NULL
)
OR (l1_batch_number = $1::BIGINT)
ORDER BY
number
LIMIT
1
"#,
first_miniblock_of_batch.0 as i64
l1_batch_number
.miniblock_l1_batch
.map(|number| i64::from(number.0))
)
.fetch_optional(self.storage.conn())
.await?
@@ -629,6 +600,7 @@ impl BlocksWeb3Dal<'_, '_> {
mod tests {
use zksync_types::{
block::{MiniblockHasher, MiniblockHeader},
snapshots::SnapshotRecoveryStatus,
MiniblockNumber, ProtocolVersion, ProtocolVersionId,
};

@@ -698,8 +670,18 @@ mod tests {
async fn resolving_earliest_block_id() {
let connection_pool = ConnectionPool::test_pool().await;
let mut conn = connection_pool.access_storage().await.unwrap();

let miniblock_number = conn
.blocks_web3_dal()
.resolve_block_id(api::BlockId::Number(api::BlockNumber::Earliest))
.await;
assert_eq!(miniblock_number.unwrap(), None);

conn.protocol_versions_dal()
.save_protocol_version_with_tx(ProtocolVersion::default())
.await;
conn.blocks_dal()
.delete_miniblocks(MiniblockNumber(0))
.insert_miniblock(&create_miniblock_header(0))
.await
.unwrap();

@@ -714,13 +696,23 @@ mod tests {
async fn resolving_latest_block_id() {
let connection_pool = ConnectionPool::test_pool().await;
let mut conn = connection_pool.access_storage().await.unwrap();
conn.blocks_dal()
.delete_miniblocks(MiniblockNumber(0))
.await
.unwrap();
conn.protocol_versions_dal()
.save_protocol_version_with_tx(ProtocolVersion::default())
.await;

let miniblock_number = conn
.blocks_web3_dal()
.resolve_block_id(api::BlockId::Number(api::BlockNumber::Latest))
.await
.unwrap();
assert_eq!(miniblock_number, None);
let miniblock_number = conn
.blocks_web3_dal()
.resolve_block_id(api::BlockId::Number(api::BlockNumber::Pending))
.await
.unwrap();
assert_eq!(miniblock_number, Some(MiniblockNumber(0)));

conn.blocks_dal()
.insert_miniblock(&create_miniblock_header(0))
.await
@@ -766,6 +758,31 @@ mod tests {
assert_eq!(miniblock_number.unwrap(), Some(MiniblockNumber(1)));
}

#[tokio::test]
async fn resolving_pending_block_id_for_snapshot_recovery() {
let connection_pool = ConnectionPool::test_pool().await;
let mut conn = connection_pool.access_storage().await.unwrap();
let snapshot_recovery = SnapshotRecoveryStatus {
l1_batch_number: L1BatchNumber(23),
l1_batch_root_hash: H256::zero(),
miniblock_number: MiniblockNumber(42),
miniblock_root_hash: H256::zero(),
last_finished_chunk_id: None,
total_chunk_count: 100,
};
conn.snapshot_recovery_dal()
.set_applied_snapshot_status(&snapshot_recovery)
.await
.unwrap();

let miniblock_number = conn
.blocks_web3_dal()
.resolve_block_id(api::BlockId::Number(api::BlockNumber::Pending))
.await
.unwrap();
assert_eq!(miniblock_number, Some(MiniblockNumber(43)));
}

#[tokio::test]
async fn resolving_block_by_hash() {
let connection_pool = ConnectionPool::test_pool().await;
Loading