From 89de7468f5d671d97423143416a69a933b1de431 Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:42:16 -0400 Subject: [PATCH 01/13] Make a crate --- rust/processor/src/grpc_stream.rs | 446 +++++++++++++++++++----------- 1 file changed, 279 insertions(+), 167 deletions(-) diff --git a/rust/processor/src/grpc_stream.rs b/rust/processor/src/grpc_stream.rs index e1674dcd0..74afeb686 100644 --- a/rust/processor/src/grpc_stream.rs +++ b/rust/processor/src/grpc_stream.rs @@ -35,6 +35,8 @@ pub const RECONNECTION_MAX_RETRIES: u64 = 5; /// 256MB pub const MAX_RESPONSE_SIZE: usize = 1024 * 1024 * 256; +const PROCESSOR_SERVICE_TYPE: &str = "processor"; + #[derive(Clone)] pub struct TransactionsPBResponse { pub transactions: Vec, @@ -296,13 +298,7 @@ pub async fn get_chain_id( } } -/// Gets a batch of transactions from the stream. Batch size is set in the grpc server. -/// The number of batches depends on our config -/// There could be several special scenarios: -/// 1. If we lose the connection, we will try reconnecting X times within Y seconds before crashing. -/// 2. If we specified an end version and we hit that, we will stop fetching, but we will make sure that -/// all existing transactions are processed -pub async fn create_fetcher_loop( +pub struct TransactionStream { txn_sender: AsyncSender, indexer_grpc_data_service_address: Url, indexer_grpc_http2_ping_interval: Duration, @@ -314,54 +310,109 @@ pub async fn create_fetcher_loop( auth_token: String, processor_name: String, transaction_filter: crate::transaction_filter::TransactionFilter, - // The number of transactions per protobuf batch pb_channel_txn_chunk_size: usize, -) { - info!( - processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - end_version = request_ending_version, - "[Parser] Connecting to GRPC stream", - ); - let mut response = get_stream( - indexer_grpc_data_service_address.clone(), - indexer_grpc_http2_ping_interval, - indexer_grpc_http2_ping_timeout, - indexer_grpc_reconnection_timeout_secs, - starting_version, - request_ending_version, - auth_token.clone(), - processor_name.to_string(), - ) - .await; - let mut connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { - Some(connection_id) => connection_id.to_str().unwrap().to_string(), - None => "".to_string(), - }; - let mut resp_stream = response.into_inner(); - info!( - processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, - start_version = starting_version, - end_version = request_ending_version, - "[Parser] Successfully connected to GRPC stream", - ); + resp_stream: Option>, + connection_id: Option, + grpc_channel_recv_latency: std::time::Instant, + next_version_to_fetch: u64, + reconnection_retries: u64, + last_fetched_version: i64, + fetch_ma: MovingAverage, + send_ma: MovingAverage, +} + +impl TransactionStream { + pub fn new( + txn_sender: AsyncSender, + indexer_grpc_data_service_address: Url, + indexer_grpc_http2_ping_interval: Duration, + indexer_grpc_http2_ping_timeout: Duration, + indexer_grpc_reconnection_timeout_secs: Duration, + indexer_grpc_response_item_timeout_secs: Duration, + starting_version: u64, + request_ending_version: Option, + auth_token: String, + processor_name: String, + transaction_filter: crate::transaction_filter::TransactionFilter, + pb_channel_txn_chunk_size: usize, + ) -> Self { + Self { + txn_sender, + indexer_grpc_data_service_address, + indexer_grpc_http2_ping_interval, + indexer_grpc_http2_ping_timeout, + indexer_grpc_reconnection_timeout_secs, + indexer_grpc_response_item_timeout_secs, + starting_version, + request_ending_version, + auth_token, + processor_name, + transaction_filter, + pb_channel_txn_chunk_size, + resp_stream: None, + connection_id: None, + grpc_channel_recv_latency: std::time::Instant::now(), + next_version_to_fetch: starting_version, + reconnection_retries: 0, + last_fetched_version: starting_version as i64 - 1, + fetch_ma: MovingAverage::new(3000), + send_ma: MovingAverage::new(3000), + } + } - let mut grpc_channel_recv_latency = std::time::Instant::now(); - let mut next_version_to_fetch = starting_version; - let mut reconnection_retries = 0; - let mut last_fetched_version = starting_version as i64 - 1; - let mut fetch_ma = MovingAverage::new(3000); - let mut send_ma = MovingAverage::new(3000); + async fn init_stream(&mut self) { + info!( + processor_name = self.processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = self.indexer_grpc_data_service_address.to_string(), + start_version = self.starting_version, + end_version = self.request_ending_version, + "[Parser] Connecting to GRPC stream", + ); + let response = get_stream( + self.indexer_grpc_data_service_address.clone(), + self.indexer_grpc_http2_ping_interval, + self.indexer_grpc_http2_ping_timeout, + self.indexer_grpc_reconnection_timeout_secs, + self.starting_version, + self.request_ending_version, + self.auth_token.clone(), + self.processor_name.to_string(), + ) + .await; + let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { + Some(connection_id) => connection_id.to_str().unwrap().to_string(), + None => "".to_string(), + }; + self.connection_id = Some(connection_id); + self.resp_stream = Some(response.into_inner()); + info!( + processor_name = self.processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, + start_version = self.starting_version, + end_version = self.request_ending_version, + "[Parser] Successfully connected to GRPC stream", + ); + } - loop { + /// Gets a batch of transactions from the stream. Batch size is set in the grpc server. + /// The number of batches depends on our config + /// There could be several special scenarios: + /// 1. If we lose the connection, we will try reconnecting X times within Y seconds before crashing. + /// 2. If we specified an end version and we hit that, we will stop fetching, but we will make sure that + /// all existing transactions are processed + /// Returns + /// - true if should continue fetching + /// - false if we reached the end of the stream or there is an error and the loop should stop + pub async fn get_next_transaction_batch(&mut self) -> bool { let is_success = match tokio::time::timeout( - indexer_grpc_response_item_timeout_secs, - resp_stream.next(), + self.indexer_grpc_response_item_timeout_secs, + self.resp_stream + .as_mut() + .expect("[Parser] GRPC stream is not initialized. Did you call init_stream?") + .next(), ) .await { @@ -369,7 +420,7 @@ pub async fn create_fetcher_loop( Ok(response) => { match response { Some(Ok(mut r)) => { - reconnection_retries = 0; + self.reconnection_retries = 0; let start_version = r.transactions.as_slice().first().unwrap().version; let start_txn_timestamp = r.transactions.as_slice().first().unwrap().timestamp.clone(); @@ -377,18 +428,20 @@ pub async fn create_fetcher_loop( let end_txn_timestamp = r.transactions.as_slice().last().unwrap().timestamp.clone(); - next_version_to_fetch = end_version + 1; + self.next_version_to_fetch = end_version + 1; let size_in_bytes = r.encoded_len() as u64; let chain_id: u64 = r.chain_id.expect("[Parser] Chain Id doesn't exist."); let num_txns = r.transactions.len(); - let duration_in_secs = grpc_channel_recv_latency.elapsed().as_secs_f64(); - fetch_ma.tick_now(num_txns as u64); + let duration_in_secs = + self.grpc_channel_recv_latency.elapsed().as_secs_f64(); + self.fetch_ma.tick_now(num_txns as u64); let num_txns = r.transactions.len(); // Filter out the txns we don't care about - r.transactions.retain(|txn| transaction_filter.include(txn)); + r.transactions + .retain(|txn| self.transaction_filter.include(txn)); let num_txn_post_filter = r.transactions.len(); let num_filtered_txns = num_txns - num_txn_post_filter; @@ -396,10 +449,10 @@ pub async fn create_fetcher_loop( let label = ProcessorStep::ReceivedTxnsFromGrpc.get_label(); info!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, start_version, end_version, start_txn_timestamp_iso = start_txn_timestamp @@ -412,32 +465,32 @@ pub async fn create_fetcher_loop( .unwrap_or_default(), num_of_transactions = end_version - start_version + 1, num_filtered_txns, - channel_size = txn_sender.len(), + channel_size = self.txn_sender.len(), size_in_bytes, duration_in_secs, - tps = fetch_ma.avg().ceil() as u64, + tps = self.fetch_ma.avg().ceil() as u64, bytes_per_sec = size_in_bytes as f64 / duration_in_secs, step, "{}", label, ); - if last_fetched_version + 1 != start_version as i64 { + if self.last_fetched_version + 1 != start_version as i64 { error!( - batch_start_version = last_fetched_version + 1, - last_fetched_version, + batch_start_version = self.last_fetched_version + 1, + self.last_fetched_version, current_fetched_version = start_version, "[Parser] Received batch with gap from GRPC stream" ); panic!("[Parser] Received batch with gap from GRPC stream"); } - last_fetched_version = end_version as i64; + self.last_fetched_version = end_version as i64; LATEST_PROCESSED_VERSION - .with_label_values(&[&processor_name, step, label, "-"]) + .with_label_values(&[&self.processor_name, step, label, "-"]) .set(end_version as i64); TRANSACTION_UNIX_TIMESTAMP - .with_label_values(&[&processor_name, step, label, "-"]) + .with_label_values(&[&self.processor_name, step, label, "-"]) .set( start_txn_timestamp .as_ref() @@ -445,16 +498,16 @@ pub async fn create_fetcher_loop( .unwrap_or_default(), ); PROCESSED_BYTES_COUNT - .with_label_values(&[&processor_name, step, label, "-"]) + .with_label_values(&[&self.processor_name, step, label, "-"]) .inc_by(size_in_bytes); NUM_TRANSACTIONS_PROCESSED_COUNT - .with_label_values(&[&processor_name, step, label, "-"]) + .with_label_values(&[&self.processor_name, step, label, "-"]) .inc_by(end_version - start_version + 1); let txn_channel_send_latency = std::time::Instant::now(); //potentially break txn_pb into many `TransactionsPBResponse` that are each `pb_channel_txn_chunk_size` txns max in size - if num_txn_post_filter < pb_channel_txn_chunk_size { + if num_txn_post_filter < self.pb_channel_txn_chunk_size { // We only need to send one; avoid the chunk/clone let txn_pb = TransactionsPBResponse { transactions: r.transactions, @@ -466,13 +519,13 @@ pub async fn create_fetcher_loop( size_in_bytes, }; - match txn_sender.send(txn_pb).await { + match self.txn_sender.send(txn_pb).await { Ok(()) => {}, Err(e) => { error!( - processor_name = processor_name, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, + processor_name = self.processor_name, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, error = ?e, "[Parser] Error sending GRPC response to channel." ); @@ -486,7 +539,7 @@ pub async fn create_fetcher_loop( let pb_txn_chunks: Vec> = r .transactions .into_iter() - .chunks(pb_channel_txn_chunk_size) + .chunks(self.pb_channel_txn_chunk_size) .into_iter() .map(|chunk| chunk.collect()) .collect(); @@ -503,13 +556,13 @@ pub async fn create_fetcher_loop( size_in_bytes, }; - match txn_sender.send(txn_pb).await { + match self.txn_sender.send(txn_pb).await { Ok(()) => {}, Err(e) => { error!( - processor_name = processor_name, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, + processor_name = self.processor_name, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, error = ?e, "[Parser] Error sending GRPC response to channel." ); @@ -520,16 +573,16 @@ pub async fn create_fetcher_loop( } let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); - send_ma.tick_now(num_txns as u64); - let tps = send_ma.avg().ceil() as u64; + self.send_ma.tick_now(num_txns as u64); + let tps = self.send_ma.avg().ceil() as u64; let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; - let channel_size = txn_sender.len(); + let channel_size = self.txn_sender.len(); debug!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, start_version, end_version, channel_size, @@ -541,24 +594,24 @@ pub async fn create_fetcher_loop( "[Parser] Successfully sent transactions to channel." ); FETCHER_THREAD_CHANNEL_SIZE - .with_label_values(&[&processor_name]) + .with_label_values(&[&self.processor_name]) .set(channel_size as i64); - grpc_channel_recv_latency = std::time::Instant::now(); + self.grpc_channel_recv_latency = std::time::Instant::now(); NUM_TRANSACTIONS_FILTERED_OUT_COUNT - .with_label_values(&[&processor_name]) + .with_label_values(&[&self.processor_name]) .inc_by(num_filtered_txns as u64); true }, // Error receiving datastream response Some(Err(rpc_error)) => { tracing::warn!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, - start_version = starting_version, - end_version = request_ending_version, + stream_address = self.indexer_grpc_data_service_address.to_string(), + self.connection_id, + start_version = self.starting_version, + end_version = self.request_ending_version, error = ?rpc_error, "[Parser] Error receiving datastream response." ); @@ -567,12 +620,12 @@ pub async fn create_fetcher_loop( // Stream is finished None => { tracing::warn!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, - start_version = starting_version, - end_version = request_ending_version, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, + start_version = self.starting_version, + end_version = self.request_ending_version, "[Parser] Stream ended." ); false @@ -582,12 +635,12 @@ pub async fn create_fetcher_loop( // Timeout receiving datastream response Err(e) => { tracing::warn!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, - start_version = starting_version, - end_version = request_ending_version, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, + start_version = self.starting_version, + end_version = self.request_ending_version, error = ?e, "[Parser] Timeout receiving datastream response." ); @@ -595,29 +648,29 @@ pub async fn create_fetcher_loop( }, }; // Check if we're at the end of the stream - let is_end = if let Some(ending_version) = request_ending_version { - next_version_to_fetch > ending_version + let is_end = if let Some(ending_version) = self.request_ending_version { + self.next_version_to_fetch > ending_version } else { false }; if is_end { info!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, - ending_version = request_ending_version, - next_version_to_fetch = next_version_to_fetch, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, + ending_version = self.request_ending_version, + next_version_to_fetch = self.next_version_to_fetch, "[Parser] Reached ending version.", ); // Wait for the fetched transactions to finish processing before closing the channel loop { - let channel_size = txn_sender.len(); + let channel_size = self.txn_sender.len(); info!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, channel_size, "[Parser] Waiting for channel to be empty" ); @@ -627,68 +680,127 @@ pub async fn create_fetcher_loop( tokio::time::sleep(Duration::from_millis(100)).await; } info!( - processor_name = processor_name, + processor_name = self.processor_name, service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, "[Parser] Transaction fetcher send channel is closed." ); - break; + + // If we're at the end of the stream, return false to signal that the fetcher should stop looping + false } else { // The rest is to see if we need to reconnect - if is_success { - continue; + if !is_success { + self.reconnect_to_grpc().await; } - // Sleep for 100ms between reconnect tries - // TODO: Turn this into exponential backoff - tokio::time::sleep(Duration::from_millis(100)).await; + true + } + } - if reconnection_retries >= RECONNECTION_MAX_RETRIES { - error!( - processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - "[Parser] Reconnected more than {RECONNECTION_MAX_RETRIES} times. Will not retry.", - ); - panic!("[Parser] Reconnected more than {RECONNECTION_MAX_RETRIES} times. Will not retry.") - } - reconnection_retries += 1; - info!( - processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - starting_version = next_version_to_fetch, - ending_version = request_ending_version, - reconnection_retries = reconnection_retries, - "[Parser] Reconnecting to GRPC stream" - ); - response = get_stream( - indexer_grpc_data_service_address.clone(), - indexer_grpc_http2_ping_interval, - indexer_grpc_http2_ping_timeout, - indexer_grpc_reconnection_timeout_secs, - next_version_to_fetch, - request_ending_version, - auth_token.clone(), - processor_name.to_string(), - ) - .await; - connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { - Some(connection_id) => connection_id.to_str().unwrap().to_string(), - None => "".to_string(), - }; - resp_stream = response.into_inner(); - info!( - processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - connection_id, - starting_version = next_version_to_fetch, - ending_version = request_ending_version, - reconnection_retries = reconnection_retries, - "[Parser] Successfully reconnected to GRPC stream" + async fn reconnect_to_grpc(&mut self) { + // Sleep for 100ms between reconnect tries + // TODO: Turn this into exponential backoff + tokio::time::sleep(Duration::from_millis(100)).await; + + if self.reconnection_retries >= RECONNECTION_MAX_RETRIES { + error!( + processor_name = self.processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = self.indexer_grpc_data_service_address.to_string(), + "[Parser] Reconnected more than 100 times. Will not retry.", ); + panic!("[Parser] Reconnected more than 100 times. Will not retry.") + } + self.reconnection_retries += 1; + info!( + processor_name = self.processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = self.indexer_grpc_data_service_address.to_string(), + starting_version = self.next_version_to_fetch, + ending_version = self.request_ending_version, + reconnection_retries = self.reconnection_retries, + "[Parser] Reconnecting to GRPC stream" + ); + let response = get_stream( + self.indexer_grpc_data_service_address.clone(), + self.indexer_grpc_http2_ping_interval, + self.indexer_grpc_http2_ping_timeout, + self.indexer_grpc_reconnection_timeout_secs, + self.next_version_to_fetch, + self.request_ending_version, + self.auth_token.clone(), + self.processor_name.to_string(), + ) + .await; + let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { + Some(connection_id) => connection_id.to_str().unwrap().to_string(), + None => "".to_string(), + }; + self.connection_id = Some(connection_id); + self.resp_stream = Some(response.into_inner()); + info!( + processor_name = self.processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = self.indexer_grpc_data_service_address.to_string(), + connection_id = self.connection_id, + starting_version = self.next_version_to_fetch, + ending_version = self.request_ending_version, + reconnection_retries = self.reconnection_retries, + "[Parser] Successfully reconnected to GRPC stream" + ); + } + + pub async fn get_chain_id(self) -> u64 { + get_chain_id( + self.indexer_grpc_data_service_address, + self.indexer_grpc_http2_ping_interval, + self.indexer_grpc_http2_ping_timeout, + self.indexer_grpc_reconnection_timeout_secs, + self.auth_token, + self.processor_name, + ) + .await + } +} + +pub async fn create_fetcher_loop( + txn_sender: AsyncSender, + indexer_grpc_data_service_address: Url, + indexer_grpc_http2_ping_interval: Duration, + indexer_grpc_http2_ping_timeout: Duration, + indexer_grpc_reconnection_timeout_secs: Duration, + indexer_grpc_response_item_timeout_secs: Duration, + starting_version: u64, + request_ending_version: Option, + auth_token: String, + processor_name: String, + transaction_filter: crate::transaction_filter::TransactionFilter, + // The number of transactions per protobuf batch + pb_channel_txn_chunk_size: usize, +) { + let mut transaction_stream = TransactionStream::new( + txn_sender, + indexer_grpc_data_service_address, + indexer_grpc_http2_ping_interval, + indexer_grpc_http2_ping_timeout, + indexer_grpc_reconnection_timeout_secs, + indexer_grpc_response_item_timeout_secs, + starting_version, + request_ending_version, + auth_token, + processor_name, + transaction_filter, + pb_channel_txn_chunk_size, + ); + + transaction_stream.init_stream().await; + + loop { + let should_continue_fetching = transaction_stream.get_next_transaction_batch().await; + if !should_continue_fetching { + break; } } } From cfe52cd40a4e8d0965be60fc838003aca0feb95e Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:44:09 -0400 Subject: [PATCH 02/13] Clean up const --- rust/processor/src/grpc_stream.rs | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/rust/processor/src/grpc_stream.rs b/rust/processor/src/grpc_stream.rs index 74afeb686..19b800c3e 100644 --- a/rust/processor/src/grpc_stream.rs +++ b/rust/processor/src/grpc_stream.rs @@ -84,7 +84,7 @@ pub async fn get_stream( ) -> Response> { info!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), start_version = starting_version, end_version = ending_version, @@ -112,7 +112,7 @@ pub async fn get_stream( info!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), start_version = starting_version, end_version = ending_version, @@ -133,7 +133,7 @@ pub async fn get_stream( Err(e) => { error!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), start_version = starting_version, end_version = ending_version, @@ -160,7 +160,7 @@ pub async fn get_stream( Err(e) => { error!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), start_version = starting_version, ending_version = ending_version, @@ -173,7 +173,7 @@ pub async fn get_stream( let count = ending_version.map(|v| (v as i64 - starting_version as i64 + 1) as u64); info!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), start_version = starting_version, end_version = ending_version, @@ -200,7 +200,7 @@ pub async fn get_stream( Err(e) => { error!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), start_version = starting_version, end_version = ending_version, @@ -222,7 +222,7 @@ pub async fn get_stream( Err(e) => { error!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), start_version = starting_version, ending_version = ending_version, @@ -244,7 +244,7 @@ pub async fn get_chain_id( ) -> u64 { info!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), "[Parser] Connecting to GRPC stream to get chain id", ); @@ -266,7 +266,7 @@ pub async fn get_chain_id( let mut resp_stream = response.into_inner(); info!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), connection_id, "[Parser] Successfully connected to GRPC stream to get chain id", @@ -277,7 +277,7 @@ pub async fn get_chain_id( Some(Err(rpc_error)) => { error!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), connection_id, error = ?rpc_error, @@ -288,7 +288,7 @@ pub async fn get_chain_id( None => { error!( processor_name = processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = indexer_grpc_data_service_address.to_string(), connection_id, "[Parser] Stream ended before getting response fo for chain id" @@ -450,7 +450,7 @@ impl TransactionStream { info!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, start_version, @@ -580,7 +580,7 @@ impl TransactionStream { let channel_size = self.txn_sender.len(); debug!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, start_version, @@ -607,7 +607,7 @@ impl TransactionStream { Some(Err(rpc_error)) => { tracing::warn!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), self.connection_id, start_version = self.starting_version, @@ -621,7 +621,7 @@ impl TransactionStream { None => { tracing::warn!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, start_version = self.starting_version, @@ -636,7 +636,7 @@ impl TransactionStream { Err(e) => { tracing::warn!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, start_version = self.starting_version, @@ -656,7 +656,7 @@ impl TransactionStream { if is_end { info!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, ending_version = self.request_ending_version, @@ -668,7 +668,7 @@ impl TransactionStream { let channel_size = self.txn_sender.len(); info!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, channel_size, @@ -681,7 +681,7 @@ impl TransactionStream { } info!( processor_name = self.processor_name, - service_type = crate::worker::PROCESSOR_SERVICE_TYPE, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, "[Parser] Transaction fetcher send channel is closed." From 675109720029ae136f6fe3c106fbf45c9195b2ed Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:59:37 -0400 Subject: [PATCH 03/13] Fix lint --- rust/processor/src/grpc_stream.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/processor/src/grpc_stream.rs b/rust/processor/src/grpc_stream.rs index 19b800c3e..5fd2151ad 100644 --- a/rust/processor/src/grpc_stream.rs +++ b/rust/processor/src/grpc_stream.rs @@ -403,6 +403,7 @@ impl TransactionStream { /// 1. If we lose the connection, we will try reconnecting X times within Y seconds before crashing. /// 2. If we specified an end version and we hit that, we will stop fetching, but we will make sure that /// all existing transactions are processed + /// /// Returns /// - true if should continue fetching /// - false if we reached the end of the stream or there is an error and the loop should stop From 4e5e1cb28c13692b34294e08fdd64df8574d9e7b Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Mon, 10 Jun 2024 20:52:05 -0400 Subject: [PATCH 04/13] Actually move it to its own crate --- rust/Cargo.lock | 32 ++++++ rust/Cargo.toml | 11 +- .../Cargo.toml | 30 ++++++ .../src/lib.rs | 2 + .../src/transaction_stream.rs} | 7 +- .../src/utils/counters.rs | 100 ++++++++++++++++++ .../src/utils/mod.rs | 2 + .../src/utils/util.rs | 28 +++++ rust/processor/Cargo.toml | 2 + rust/processor/src/config.rs | 4 +- rust/processor/src/lib.rs | 2 - rust/processor/src/worker.rs | 8 +- rust/transaction-filter/Cargo.toml | 18 ++++ rust/transaction-filter/src/lib.rs | 1 + .../src/transaction_filter.rs | 0 15 files changed, 235 insertions(+), 12 deletions(-) create mode 100644 rust/aptos-indexer-transaction-stream/Cargo.toml create mode 100644 rust/aptos-indexer-transaction-stream/src/lib.rs rename rust/{processor/src/grpc_stream.rs => aptos-indexer-transaction-stream/src/transaction_stream.rs} (99%) create mode 100644 rust/aptos-indexer-transaction-stream/src/utils/counters.rs create mode 100644 rust/aptos-indexer-transaction-stream/src/utils/mod.rs create mode 100644 rust/aptos-indexer-transaction-stream/src/utils/util.rs create mode 100644 rust/transaction-filter/Cargo.toml create mode 100644 rust/transaction-filter/src/lib.rs rename rust/{processor => transaction-filter}/src/transaction_filter.rs (100%) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index acb7e643c..7432c4d71 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -116,6 +116,27 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "aptos-indexer-transaction-stream" +version = "0.1.0" +dependencies = [ + "aptos-moving-average", + "aptos-protos", + "bigdecimal", + "chrono", + "futures-util", + "itertools 0.12.1", + "kanal", + "once_cell", + "prometheus", + "prost 0.12.3", + "tokio", + "tonic 0.11.0", + "tracing", + "transaction-filter", + "url", +] + [[package]] name = "aptos-moving-average" version = "0.1.0" @@ -2191,6 +2212,7 @@ version = "1.0.0" dependencies = [ "ahash", "anyhow", + "aptos-indexer-transaction-stream", "aptos-moving-average", "aptos-protos", "async-trait", @@ -2234,6 +2256,7 @@ dependencies = [ "tokio-postgres", "tonic 0.11.0", "tracing", + "transaction-filter", "unescape", "url", ] @@ -3614,6 +3637,15 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "transaction-filter" +version = "0.1.0" +dependencies = [ + "ahash", + "aptos-protos", + "serde", +] + [[package]] name = "try-lock" version = "0.2.4" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2f984cd89..0642e0ca3 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,7 +1,14 @@ [workspace] resolver = "2" -members = ["indexer-metrics", "moving-average", "processor", "server-framework"] +members = [ + "aptos-indexer-transaction-stream", + "indexer-metrics", + "moving-average", + "processor", + "server-framework", + "transaction-filter", +] [workspace.package] authors = ["Aptos Labs "] @@ -16,6 +23,8 @@ rust-version = "1.75" processor = { path = "processor" } server-framework = { path = "server-framework" } aptos-moving-average = { path = "moving-average" } +aptos-indexer-transaction-stream = { path = "aptos-indexer-transaction-stream" } +transaction-filter = { path = "transaction-filter" } ahash = { version = "0.8.7", features = ["serde"] } anyhow = "1.0.62" diff --git a/rust/aptos-indexer-transaction-stream/Cargo.toml b/rust/aptos-indexer-transaction-stream/Cargo.toml new file mode 100644 index 000000000..0a7b5db97 --- /dev/null +++ b/rust/aptos-indexer-transaction-stream/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "aptos-indexer-transaction-stream" +version = "0.1.0" + +# Workspace inherited keys +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +aptos-moving-average = { workspace = true } +aptos-protos = { workspace = true } +bigdecimal = { workspace = true } +chrono = { workspace = true } +futures-util = { workspace = true } +itertools = { workspace = true } +kanal = { workspace = true } +once_cell = { workspace = true } +prometheus = { workspace = true } +prost = { workspace = true } +tokio = { workspace = true } +tonic = { workspace = true } +tracing = { workspace = true } +transaction-filter = { workspace = true } +url = { workspace = true } diff --git a/rust/aptos-indexer-transaction-stream/src/lib.rs b/rust/aptos-indexer-transaction-stream/src/lib.rs new file mode 100644 index 000000000..d8a510f16 --- /dev/null +++ b/rust/aptos-indexer-transaction-stream/src/lib.rs @@ -0,0 +1,2 @@ +pub mod transaction_stream; +pub mod utils; diff --git a/rust/processor/src/grpc_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs similarity index 99% rename from rust/processor/src/grpc_stream.rs rename to rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 5fd2151ad..2c902b255 100644 --- a/rust/processor/src/grpc_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -21,6 +21,7 @@ use std::time::Duration; use tokio::time::timeout; use tonic::{Response, Streaming}; use tracing::{debug, error, info}; +use transaction_filter::transaction_filter::TransactionFilter; use url::Url; /// GRPC request metadata key for the token ID. @@ -309,7 +310,7 @@ pub struct TransactionStream { request_ending_version: Option, auth_token: String, processor_name: String, - transaction_filter: crate::transaction_filter::TransactionFilter, + transaction_filter: TransactionFilter, pb_channel_txn_chunk_size: usize, resp_stream: Option>, connection_id: Option, @@ -333,7 +334,7 @@ impl TransactionStream { request_ending_version: Option, auth_token: String, processor_name: String, - transaction_filter: crate::transaction_filter::TransactionFilter, + transaction_filter: TransactionFilter, pb_channel_txn_chunk_size: usize, ) -> Self { Self { @@ -777,7 +778,7 @@ pub async fn create_fetcher_loop( request_ending_version: Option, auth_token: String, processor_name: String, - transaction_filter: crate::transaction_filter::TransactionFilter, + transaction_filter: TransactionFilter, // The number of transactions per protobuf batch pb_channel_txn_chunk_size: usize, ) { diff --git a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs new file mode 100644 index 000000000..e1b613d85 --- /dev/null +++ b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs @@ -0,0 +1,100 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use once_cell::sync::Lazy; +use prometheus::{ + register_gauge_vec, register_int_counter_vec, register_int_gauge_vec, GaugeVec, IntCounterVec, + IntGaugeVec, +}; + +pub enum ProcessorStep { + ReceivedTxnsFromGrpc, + // Received transactions from GRPC. Sending transactions to channel. + ProcessedBatch, + // Processor finished processing one batch of transaction + ProcessedMultipleBatches, // Processor finished processing multiple batches of transactions +} + +impl ProcessorStep { + pub fn get_step(&self) -> &'static str { + match self { + ProcessorStep::ReceivedTxnsFromGrpc => "1", + ProcessorStep::ProcessedBatch => "2", + ProcessorStep::ProcessedMultipleBatches => "3", + } + } + + pub fn get_label(&self) -> &'static str { + match self { + ProcessorStep::ReceivedTxnsFromGrpc => { + "[Parser] Received transactions from GRPC. Sending transactions to channel." + }, + ProcessorStep::ProcessedBatch => { + "[Parser] Processor finished processing one batch of transaction" + }, + ProcessorStep::ProcessedMultipleBatches => { + "[Parser] Processor finished processing multiple batches of transactions" + }, + } + } +} + +/// Max version processed +pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { + register_int_gauge_vec!( + "indexer_processor_latest_version", + "Latest version a processor has fully consumed", + &["processor_name", "step", "message", "task_index"] + ) + .unwrap() +}); + +/// Count of bytes processed. +pub static PROCESSED_BYTES_COUNT: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "indexer_processor_processed_bytes_count", + "Count of bytes processed", + &["processor_name", "step", "message", "task_index"] + ) + .unwrap() +}); + +/// Count of transactions processed. +pub static NUM_TRANSACTIONS_PROCESSED_COUNT: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "indexer_processor_num_transactions_processed_count", + "Number of transactions processed", + &["processor_name", "step", "message", "task_index"] + ) + .unwrap() +}); + +/// Count of transactions filtered out +pub static NUM_TRANSACTIONS_FILTERED_OUT_COUNT: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "indexer_processor_num_transactions_filtered_out_count", + "Number of transactions filtered out", + &["processor_name"] + ) + .unwrap() +}); + +/// Size of the channel containing transactions fetched from GRPC, waiting to be processed +pub static FETCHER_THREAD_CHANNEL_SIZE: Lazy = Lazy::new(|| { + register_int_gauge_vec!( + "indexer_processor_fetcher_thread_channel_size", + "Size of the fetcher thread channel", + &["processor_name"] + ) + .unwrap() +}); + +/// Transaction timestamp in unixtime +pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { + register_gauge_vec!( + "indexer_processor_transaction_unix_timestamp", + "Transaction timestamp in unixtime", + &["processor_name", "step", "message", "task_index"] + ) + .unwrap() +}); diff --git a/rust/aptos-indexer-transaction-stream/src/utils/mod.rs b/rust/aptos-indexer-transaction-stream/src/utils/mod.rs new file mode 100644 index 000000000..78b4ba670 --- /dev/null +++ b/rust/aptos-indexer-transaction-stream/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod counters; +pub mod util; diff --git a/rust/aptos-indexer-transaction-stream/src/utils/util.rs b/rust/aptos-indexer-transaction-stream/src/utils/util.rs new file mode 100644 index 000000000..5117c2615 --- /dev/null +++ b/rust/aptos-indexer-transaction-stream/src/utils/util.rs @@ -0,0 +1,28 @@ +use aptos_protos::util::timestamp::Timestamp; + +// 9999-12-31 23:59:59, this is the max supported by Google BigQuery +pub const MAX_TIMESTAMP_SECS: i64 = 253_402_300_799; + +pub fn parse_timestamp(ts: &Timestamp, version: i64) -> chrono::NaiveDateTime { + let final_ts = if ts.seconds >= MAX_TIMESTAMP_SECS { + Timestamp { + seconds: MAX_TIMESTAMP_SECS, + nanos: 0, + } + } else { + ts.clone() + }; + chrono::NaiveDateTime::from_timestamp_opt(final_ts.seconds, final_ts.nanos as u32) + .unwrap_or_else(|| panic!("Could not parse timestamp {:?} for version {}", ts, version)) +} + +/// Convert the protobuf timestamp to ISO format +pub fn timestamp_to_iso(timestamp: &Timestamp) -> String { + let dt = parse_timestamp(timestamp, 0); + dt.format("%Y-%m-%dT%H:%M:%S%.9fZ").to_string() +} + +/// Convert the protobuf timestamp to unixtime +pub fn timestamp_to_unixtime(timestamp: &Timestamp) -> f64 { + timestamp.seconds as f64 + timestamp.nanos as f64 * 1e-9 +} diff --git a/rust/processor/Cargo.toml b/rust/processor/Cargo.toml index 96628d6b3..86c649292 100644 --- a/rust/processor/Cargo.toml +++ b/rust/processor/Cargo.toml @@ -15,6 +15,7 @@ rust-version = { workspace = true } [dependencies] ahash = { workspace = true } anyhow = { workspace = true } +aptos-indexer-transaction-stream = { workspace = true } aptos-moving-average = { workspace = true } aptos-protos = { workspace = true } async-trait = { workspace = true } @@ -53,6 +54,7 @@ strum = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } +transaction-filter = { workspace = true } unescape = { workspace = true } url = { workspace = true } diff --git a/rust/processor/src/config.rs b/rust/processor/src/config.rs index 5a9626b2a..2f2a47fc5 100644 --- a/rust/processor/src/config.rs +++ b/rust/processor/src/config.rs @@ -2,14 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - gap_detector::DEFAULT_GAP_DETECTION_BATCH_SIZE, processors::ProcessorConfig, - transaction_filter::TransactionFilter, worker::Worker, + gap_detector::DEFAULT_GAP_DETECTION_BATCH_SIZE, processors::ProcessorConfig, worker::Worker, }; use ahash::AHashMap; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use server_framework::RunnableConfig; use std::time::Duration; +use transaction_filter::transaction_filter::TransactionFilter; use url::Url; pub const QUERY_DEFAULT_RETRIES: u32 = 5; diff --git a/rust/processor/src/lib.rs b/rust/processor/src/lib.rs index ecb81ff3b..6142bc060 100644 --- a/rust/processor/src/lib.rs +++ b/rust/processor/src/lib.rs @@ -15,10 +15,8 @@ pub use config::IndexerGrpcProcessorConfig; mod config; pub mod gap_detector; -pub mod grpc_stream; pub mod models; pub mod processors; pub mod schema; -pub mod transaction_filter; pub mod utils; pub mod worker; diff --git a/rust/processor/src/worker.rs b/rust/processor/src/worker.rs index 06ec9b213..dbf3f8c78 100644 --- a/rust/processor/src/worker.rs +++ b/rust/processor/src/worker.rs @@ -3,7 +3,6 @@ use crate::{ config::IndexerGrpcHttp2Config, - grpc_stream::TransactionsPBResponse, models::{ledger_info::LedgerInfo, processor_status::ProcessorStatusQuery}, processors::{ account_transactions_processor::AccountTransactionsProcessor, ans_processor::AnsProcessor, @@ -17,7 +16,6 @@ use crate::{ ProcessorConfig, ProcessorTrait, }, schema::ledger_infos, - transaction_filter::TransactionFilter, utils::{ counters::{ ProcessorStep, GRPC_LATENCY_BY_PROCESSOR_IN_SECS, LATEST_PROCESSED_VERSION, @@ -34,9 +32,11 @@ use crate::{ }; use ahash::AHashMap; use anyhow::{Context, Result}; +use aptos_indexer_transaction_stream::transaction_stream::TransactionsPBResponse; use aptos_moving_average::MovingAverage; use tokio::task::JoinHandle; use tracing::{debug, error, info}; +use transaction_filter::transaction_filter::TransactionFilter; use url::Url; // this is how large the fetch queue should be. Each bucket should have a max of 80MB or so, so a batch @@ -171,7 +171,7 @@ impl Worker { let concurrent_tasks = self.number_concurrent_processing_tasks; // get the chain id - let chain_id = crate::grpc_stream::get_chain_id( + let chain_id = aptos_indexer_transaction_stream::transaction_stream::get_chain_id( self.indexer_grpc_data_service_address.clone(), self.grpc_http2_config.grpc_http2_ping_interval_in_secs(), self.grpc_http2_config.grpc_http2_ping_timeout_in_secs(), @@ -214,7 +214,7 @@ impl Worker { "[Parser] Starting fetcher thread" ); - crate::grpc_stream::create_fetcher_loop( + aptos_indexer_transaction_stream::transaction_stream::create_fetcher_loop( tx.clone(), indexer_grpc_data_service_address.clone(), indexer_grpc_http2_ping_interval, diff --git a/rust/transaction-filter/Cargo.toml b/rust/transaction-filter/Cargo.toml new file mode 100644 index 000000000..837128093 --- /dev/null +++ b/rust/transaction-filter/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "transaction-filter" +version = "0.1.0" + +# Workspace inherited keys +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ahash = { workspace = true } +aptos-protos = { workspace = true } +serde = { workspace = true } diff --git a/rust/transaction-filter/src/lib.rs b/rust/transaction-filter/src/lib.rs new file mode 100644 index 000000000..c60bee1bc --- /dev/null +++ b/rust/transaction-filter/src/lib.rs @@ -0,0 +1 @@ +pub mod transaction_filter; diff --git a/rust/processor/src/transaction_filter.rs b/rust/transaction-filter/src/transaction_filter.rs similarity index 100% rename from rust/processor/src/transaction_filter.rs rename to rust/transaction-filter/src/transaction_filter.rs From fe52037dc8e8a786dac4e3fe1e5dbf3bf273dda5 Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Tue, 11 Jun 2024 20:51:00 -0400 Subject: [PATCH 05/13] Move txn_sender out of TransactionStream struct --- .../src/transaction_stream.rs | 210 +++++++++--------- .../src/utils/counters.rs | 30 +-- rust/processor/src/utils/counters.rs | 28 +-- 3 files changed, 132 insertions(+), 136 deletions(-) diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 2c902b255..eeb39ed3d 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -300,7 +300,6 @@ pub async fn get_chain_id( } pub struct TransactionStream { - txn_sender: AsyncSender, indexer_grpc_data_service_address: Url, indexer_grpc_http2_ping_interval: Duration, indexer_grpc_http2_ping_timeout: Duration, @@ -314,17 +313,19 @@ pub struct TransactionStream { pb_channel_txn_chunk_size: usize, resp_stream: Option>, connection_id: Option, - grpc_channel_recv_latency: std::time::Instant, next_version_to_fetch: u64, reconnection_retries: u64, last_fetched_version: i64, fetch_ma: MovingAverage, - send_ma: MovingAverage, +} + +pub struct TransactionStreamOutput { + pub transactions: Vec, + pub should_continue_fetching: bool, } impl TransactionStream { pub fn new( - txn_sender: AsyncSender, indexer_grpc_data_service_address: Url, indexer_grpc_http2_ping_interval: Duration, indexer_grpc_http2_ping_timeout: Duration, @@ -338,7 +339,6 @@ impl TransactionStream { pb_channel_txn_chunk_size: usize, ) -> Self { Self { - txn_sender, indexer_grpc_data_service_address, indexer_grpc_http2_ping_interval, indexer_grpc_http2_ping_timeout, @@ -352,12 +352,10 @@ impl TransactionStream { pb_channel_txn_chunk_size, resp_stream: None, connection_id: None, - grpc_channel_recv_latency: std::time::Instant::now(), next_version_to_fetch: starting_version, reconnection_retries: 0, last_fetched_version: starting_version as i64 - 1, fetch_ma: MovingAverage::new(3000), - send_ma: MovingAverage::new(3000), } } @@ -408,7 +406,10 @@ impl TransactionStream { /// Returns /// - true if should continue fetching /// - false if we reached the end of the stream or there is an error and the loop should stop - pub async fn get_next_transaction_batch(&mut self) -> bool { + pub async fn get_next_transaction_batch(&mut self) -> TransactionStreamOutput { + let grpc_channel_recv_latency = std::time::Instant::now(); + let mut transaction_pb_response = vec![]; + let is_success = match tokio::time::timeout( self.indexer_grpc_response_item_timeout_secs, self.resp_stream @@ -435,12 +436,9 @@ impl TransactionStream { let size_in_bytes = r.encoded_len() as u64; let chain_id: u64 = r.chain_id.expect("[Parser] Chain Id doesn't exist."); let num_txns = r.transactions.len(); - let duration_in_secs = - self.grpc_channel_recv_latency.elapsed().as_secs_f64(); + let duration_in_secs = grpc_channel_recv_latency.elapsed().as_secs_f64(); self.fetch_ma.tick_now(num_txns as u64); - let num_txns = r.transactions.len(); - // Filter out the txns we don't care about r.transactions .retain(|txn| self.transaction_filter.include(txn)); @@ -467,7 +465,6 @@ impl TransactionStream { .unwrap_or_default(), num_of_transactions = end_version - start_version + 1, num_filtered_txns, - channel_size = self.txn_sender.len(), size_in_bytes, duration_in_secs, tps = self.fetch_ma.avg().ceil() as u64, @@ -506,8 +503,6 @@ impl TransactionStream { .with_label_values(&[&self.processor_name, step, label, "-"]) .inc_by(end_version - start_version + 1); - let txn_channel_send_latency = std::time::Instant::now(); - //potentially break txn_pb into many `TransactionsPBResponse` that are each `pb_channel_txn_chunk_size` txns max in size if num_txn_post_filter < self.pb_channel_txn_chunk_size { // We only need to send one; avoid the chunk/clone @@ -520,20 +515,7 @@ impl TransactionStream { end_txn_timestamp, size_in_bytes, }; - - match self.txn_sender.send(txn_pb).await { - Ok(()) => {}, - Err(e) => { - error!( - processor_name = self.processor_name, - stream_address = self.indexer_grpc_data_service_address.to_string(), - connection_id = self.connection_id, - error = ?e, - "[Parser] Error sending GRPC response to channel." - ); - panic!("[Parser] Error sending GRPC response to channel.") - }, - } + transaction_pb_response.push(txn_pb); } else { // We are breaking down a big batch into small batches; this involves an iterator let average_size_in_bytes = size_in_bytes / num_txns as u64; @@ -557,49 +539,10 @@ impl TransactionStream { end_txn_timestamp: end_txn_timestamp.clone(), size_in_bytes, }; - - match self.txn_sender.send(txn_pb).await { - Ok(()) => {}, - Err(e) => { - error!( - processor_name = self.processor_name, - stream_address = self.indexer_grpc_data_service_address.to_string(), - connection_id = self.connection_id, - error = ?e, - "[Parser] Error sending GRPC response to channel." - ); - panic!("[Parser] Error sending GRPC response to channel.") - }, - } + transaction_pb_response.push(txn_pb); } } - let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); - self.send_ma.tick_now(num_txns as u64); - let tps = self.send_ma.avg().ceil() as u64; - let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; - - let channel_size = self.txn_sender.len(); - debug!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), - connection_id = self.connection_id, - start_version, - end_version, - channel_size, - size_in_bytes, - duration_in_secs, - bytes_per_sec, - tps, - num_filtered_txns, - "[Parser] Successfully sent transactions to channel." - ); - FETCHER_THREAD_CHANNEL_SIZE - .with_label_values(&[&self.processor_name]) - .set(channel_size as i64); - self.grpc_channel_recv_latency = std::time::Instant::now(); - NUM_TRANSACTIONS_FILTERED_OUT_COUNT .with_label_values(&[&self.processor_name]) .inc_by(num_filtered_txns as u64); @@ -655,6 +598,7 @@ impl TransactionStream { } else { false }; + if is_end { info!( processor_name = self.processor_name, @@ -665,39 +609,21 @@ impl TransactionStream { next_version_to_fetch = self.next_version_to_fetch, "[Parser] Reached ending version.", ); - // Wait for the fetched transactions to finish processing before closing the channel - loop { - let channel_size = self.txn_sender.len(); - info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), - connection_id = self.connection_id, - channel_size, - "[Parser] Waiting for channel to be empty" - ); - if channel_size.is_zero() { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), - connection_id = self.connection_id, - "[Parser] Transaction fetcher send channel is closed." - ); - // If we're at the end of the stream, return false to signal that the fetcher should stop looping - false + TransactionStreamOutput { + transactions: transaction_pb_response, + should_continue_fetching: false, + } } else { // The rest is to see if we need to reconnect if !is_success { self.reconnect_to_grpc().await; } - true + TransactionStreamOutput { + transactions: transaction_pb_response, + should_continue_fetching: true, + } } } @@ -783,8 +709,7 @@ pub async fn create_fetcher_loop( pb_channel_txn_chunk_size: usize, ) { let mut transaction_stream = TransactionStream::new( - txn_sender, - indexer_grpc_data_service_address, + indexer_grpc_data_service_address.clone(), indexer_grpc_http2_ping_interval, indexer_grpc_http2_ping_timeout, indexer_grpc_reconnection_timeout_secs, @@ -792,16 +717,101 @@ pub async fn create_fetcher_loop( starting_version, request_ending_version, auth_token, - processor_name, + processor_name.clone(), transaction_filter, pb_channel_txn_chunk_size, ); transaction_stream.init_stream().await; + let mut send_ma = MovingAverage::new(3000); loop { - let should_continue_fetching = transaction_stream.get_next_transaction_batch().await; - if !should_continue_fetching { + let transactions_output = transaction_stream.get_next_transaction_batch().await; + let start_version = transactions_output + .transactions + .first() + .unwrap() + .transactions + .first() + .unwrap() + .version; + let end_version = transactions_output + .transactions + .last() + .unwrap() + .transactions + .last() + .unwrap() + .version; + let size_in_bytes = transactions_output + .transactions + .iter() + .map(|txn_pb| txn_pb.size_in_bytes) + .sum::(); + + let txn_channel_send_latency = std::time::Instant::now(); + for txn_pb in transactions_output.transactions { + match txn_sender.send(txn_pb).await { + Ok(()) => {}, + Err(e) => { + error!( + processor_name = processor_name, + stream_address = indexer_grpc_data_service_address.to_string(), + error = ?e, + "[Parser] Error sending GRPC response to channel." + ); + panic!("[Parser] Error sending GRPC response to channel.") + }, + } + } + + // Log channel send metrics + let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); + let num_txns = end_version - start_version + 1; + + send_ma.tick_now(num_txns as u64); + let tps = send_ma.avg().ceil() as u64; + let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; + let channel_size = txn_sender.len(); + debug!( + processor_name.service_type = PROCESSOR_SERVICE_TYPE, + stream_address = indexer_grpc_data_service_address.to_string(), + start_version, + end_version, + channel_size, + size_in_bytes, + duration_in_secs, + bytes_per_sec, + tps, + "[Parser] Successfully sent transactions to channel." + ); + FETCHER_THREAD_CHANNEL_SIZE + .with_label_values(&[&processor_name]) + .set(channel_size as i64); + + // Handle reaching end of stream + if !transactions_output.should_continue_fetching { + // Wait for the fetched transactions to finish processing before closing the channel + loop { + let channel_size = txn_sender.len(); + info!( + processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = indexer_grpc_data_service_address.to_string(), + channel_size, + "[Parser] Waiting for channel to be empty" + ); + if channel_size.is_zero() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + info!( + processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = indexer_grpc_data_service_address.to_string(), + "[Parser] Transaction fetcher send channel is closed." + ); break; } } diff --git a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs index e1b613d85..ac7fecaac 100644 --- a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs +++ b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs @@ -39,10 +39,14 @@ impl ProcessorStep { } } +/// These metrics are temporary (suffixed with _temp to avoid conflict with metrics in processor crate) +/// They're only defined in this crate for backwards compatibility before we migrate over to +/// using the instrumentation provided by SDK + /// Max version processed pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { register_int_gauge_vec!( - "indexer_processor_latest_version", + "indexer_processor_latest_version_temp", "Latest version a processor has fully consumed", &["processor_name", "step", "message", "task_index"] ) @@ -52,7 +56,7 @@ pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { /// Count of bytes processed. pub static PROCESSED_BYTES_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( - "indexer_processor_processed_bytes_count", + "indexer_processor_processed_bytes_count_temp", "Count of bytes processed", &["processor_name", "step", "message", "task_index"] ) @@ -62,13 +66,23 @@ pub static PROCESSED_BYTES_COUNT: Lazy = Lazy::new(|| { /// Count of transactions processed. pub static NUM_TRANSACTIONS_PROCESSED_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( - "indexer_processor_num_transactions_processed_count", + "indexer_processor_num_transactions_processed_count_temp", "Number of transactions processed", &["processor_name", "step", "message", "task_index"] ) .unwrap() }); +/// Transaction timestamp in unixtime +pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { + register_gauge_vec!( + "indexer_processor_transaction_unix_timestamp_temp", + "Transaction timestamp in unixtime", + &["processor_name", "step", "message", "task_index"] + ) + .unwrap() +}); + /// Count of transactions filtered out pub static NUM_TRANSACTIONS_FILTERED_OUT_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( @@ -88,13 +102,3 @@ pub static FETCHER_THREAD_CHANNEL_SIZE: Lazy = Lazy::new(|| { ) .unwrap() }); - -/// Transaction timestamp in unixtime -pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { - register_gauge_vec!( - "indexer_processor_transaction_unix_timestamp", - "Transaction timestamp in unixtime", - &["processor_name", "step", "message", "task_index"] - ) - .unwrap() -}); diff --git a/rust/processor/src/utils/counters.rs b/rust/processor/src/utils/counters.rs index ee9431a8d..f1fed7266 100644 --- a/rust/processor/src/utils/counters.rs +++ b/rust/processor/src/utils/counters.rs @@ -167,26 +167,6 @@ pub static NUM_TRANSACTIONS_PROCESSED_COUNT: Lazy = Lazy::new(|| .unwrap() }); -/// Count of transactions filtered out -pub static NUM_TRANSACTIONS_FILTERED_OUT_COUNT: Lazy = Lazy::new(|| { - register_int_counter_vec!( - "indexer_processor_num_transactions_filtered_out_count", - "Number of transactions filtered out", - &["processor_name"] - ) - .unwrap() -}); - -/// Size of the channel containing transactions fetched from GRPC, waiting to be processed -pub static FETCHER_THREAD_CHANNEL_SIZE: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "indexer_processor_fetcher_thread_channel_size", - "Size of the fetcher thread channel", - &["processor_name"] - ) - .unwrap() -}); - /// Overall processing time for a single batch of transactions (per task) pub static SINGLE_BATCH_PROCESSING_TIME_IN_SECS: Lazy = Lazy::new(|| { register_gauge_vec!( @@ -229,9 +209,11 @@ pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { /// Data gap warnings pub static PROCESSOR_DATA_GAP_COUNT: Lazy = Lazy::new(|| { - register_int_gauge_vec!("indexer_processor_data_gap_count", "Data gap count", &[ - "processor_name" - ]) + register_int_gauge_vec!( + "indexer_processor_data_gap_count", + "Data gap count", + &["processor_name"] + ) .unwrap() }); From d15c17d7559a75c9c69c25dfd987af83e64b42a1 Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:18:00 -0400 Subject: [PATCH 06/13] make methods pub --- .../src/transaction_stream.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index eeb39ed3d..656b0551b 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -359,7 +359,7 @@ impl TransactionStream { } } - async fn init_stream(&mut self) { + pub async fn init_stream(&mut self) { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, @@ -627,7 +627,7 @@ impl TransactionStream { } } - async fn reconnect_to_grpc(&mut self) { + pub async fn reconnect_to_grpc(&mut self) { // Sleep for 100ms between reconnect tries // TODO: Turn this into exponential backoff tokio::time::sleep(Duration::from_millis(100)).await; From 93294ffa180903efd7ae03327676c81dc7f6c31d Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:37:07 -0400 Subject: [PATCH 07/13] TransactionStreamConfig --- rust/Cargo.lock | 2 + .../Cargo.toml | 2 + .../src/config.rs | 59 +++ .../src/lib.rs | 4 + .../src/transaction_stream.rs | 419 ++++++------------ .../src/utils/counters.rs | 34 +- rust/processor/src/config.rs | 43 +- rust/processor/src/utils/counters.rs | 10 + rust/processor/src/worker.rs | 192 ++++++-- 9 files changed, 410 insertions(+), 355 deletions(-) create mode 100644 rust/aptos-indexer-transaction-stream/src/config.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 7432c4d71..5e8971a86 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -120,6 +120,7 @@ checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" name = "aptos-indexer-transaction-stream" version = "0.1.0" dependencies = [ + "anyhow", "aptos-moving-average", "aptos-protos", "bigdecimal", @@ -130,6 +131,7 @@ dependencies = [ "once_cell", "prometheus", "prost 0.12.3", + "serde", "tokio", "tonic 0.11.0", "tracing", diff --git a/rust/aptos-indexer-transaction-stream/Cargo.toml b/rust/aptos-indexer-transaction-stream/Cargo.toml index 0a7b5db97..cb31966d9 100644 --- a/rust/aptos-indexer-transaction-stream/Cargo.toml +++ b/rust/aptos-indexer-transaction-stream/Cargo.toml @@ -13,6 +13,7 @@ rust-version = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = { workspace = true } aptos-moving-average = { workspace = true } aptos-protos = { workspace = true } bigdecimal = { workspace = true } @@ -23,6 +24,7 @@ kanal = { workspace = true } once_cell = { workspace = true } prometheus = { workspace = true } prost = { workspace = true } +serde = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } diff --git a/rust/aptos-indexer-transaction-stream/src/config.rs b/rust/aptos-indexer-transaction-stream/src/config.rs new file mode 100644 index 000000000..b1ae68260 --- /dev/null +++ b/rust/aptos-indexer-transaction-stream/src/config.rs @@ -0,0 +1,59 @@ +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use url::Url; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TransactionStreamConfig { + pub indexer_grpc_data_service_address: Url, + pub starting_version: u64, + pub request_ending_version: Option, + pub auth_token: String, + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_http2_ping_interval")] + pub indexer_grpc_http2_ping_interval_secs: u64, + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_http2_ping_timeout")] + pub indexer_grpc_http2_ping_timeout_secs: u64, + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_reconnection_timeout")] + pub indexer_grpc_reconnection_timeout_secs: u64, + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_response_item_timeout")] + pub indexer_grpc_response_item_timeout_secs: u64, +} + +impl TransactionStreamConfig { + pub const fn indexer_grpc_http2_ping_interval(&self) -> Duration { + Duration::from_secs(self.indexer_grpc_http2_ping_interval_secs) + } + + pub const fn indexer_grpc_http2_ping_timeout(&self) -> Duration { + Duration::from_secs(self.indexer_grpc_http2_ping_timeout_secs) + } + + pub const fn indexer_grpc_reconnection_timeout(&self) -> Duration { + Duration::from_secs(self.indexer_grpc_reconnection_timeout_secs) + } + + pub const fn indexer_grpc_response_item_timeout(&self) -> Duration { + Duration::from_secs(self.indexer_grpc_response_item_timeout_secs) + } + + /// Indexer GRPC http2 ping interval in seconds. Defaults to 30. + /// Tonic ref: https://docs.rs/tonic/latest/tonic/transport/channel/struct.Endpoint.html#method.http2_keep_alive_interval + pub const fn default_indexer_grpc_http2_ping_interval() -> u64 { + 30 + } + + /// Indexer GRPC http2 ping timeout in seconds. Defaults to 10. + pub const fn default_indexer_grpc_http2_ping_timeout() -> u64 { + 10 + } + + /// Default timeout for establishing a grpc connection. Defaults to 5 seconds. + pub const fn default_indexer_grpc_reconnection_timeout() -> u64 { + 5 + } + + /// Default timeout for receiving an item from grpc stream. Defaults to 60 seconds. + pub const fn default_indexer_grpc_response_item_timeout() -> u64 { + 60 + } +} diff --git a/rust/aptos-indexer-transaction-stream/src/lib.rs b/rust/aptos-indexer-transaction-stream/src/lib.rs index d8a510f16..336759f6f 100644 --- a/rust/aptos-indexer-transaction-stream/src/lib.rs +++ b/rust/aptos-indexer-transaction-stream/src/lib.rs @@ -1,2 +1,6 @@ +pub mod config; pub mod transaction_stream; pub mod utils; + +pub use config::TransactionStreamConfig; +pub use transaction_stream::{TransactionStream, TransactionStreamOutput, TransactionsPBResponse}; diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 656b0551b..57a5320b0 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -1,28 +1,26 @@ +use crate::config::TransactionStreamConfig; use crate::utils::{ counters::{ - ProcessorStep, FETCHER_THREAD_CHANNEL_SIZE, LATEST_PROCESSED_VERSION, - NUM_TRANSACTIONS_FILTERED_OUT_COUNT, NUM_TRANSACTIONS_PROCESSED_COUNT, - PROCESSED_BYTES_COUNT, TRANSACTION_UNIX_TIMESTAMP, + ProcessorStep, LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_FILTERED_OUT_COUNT, + NUM_TRANSACTIONS_PROCESSED_COUNT, PROCESSED_BYTES_COUNT, TRANSACTION_UNIX_TIMESTAMP, }, util::{timestamp_to_iso, timestamp_to_unixtime}, }; +use anyhow::Result; use aptos_moving_average::MovingAverage; use aptos_protos::{ indexer::v1::{raw_data_client::RawDataClient, GetTransactionsRequest, TransactionsResponse}, transaction::v1::Transaction, util::timestamp::Timestamp, }; -use bigdecimal::Zero; use futures_util::StreamExt; use itertools::Itertools; -use kanal::AsyncSender; use prost::Message; use std::time::Duration; use tokio::time::timeout; use tonic::{Response, Streaming}; -use tracing::{debug, error, info}; +use tracing::{error, info}; use transaction_filter::transaction_filter::TransactionFilter; -use url::Url; /// GRPC request metadata key for the token ID. const GRPC_API_GATEWAY_API_KEY_HEADER: &str = "authorization"; @@ -74,35 +72,37 @@ pub fn grpc_request_builder( } pub async fn get_stream( - indexer_grpc_data_service_address: Url, - indexer_grpc_http2_ping_interval: Duration, - indexer_grpc_http2_ping_timeout: Duration, - indexer_grpc_reconnection_timeout_secs: Duration, - starting_version: u64, - ending_version: Option, - auth_token: String, + transaction_stream_config: TransactionStreamConfig, processor_name: String, ) -> Response> { info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - end_version = ending_version, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, "[Parser] Setting up rpc channel" ); let channel = tonic::transport::Channel::from_shared( - indexer_grpc_data_service_address.to_string(), + transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), ) .expect( "[Parser] Failed to build GRPC channel, perhaps because the data service URL is invalid", ) - .http2_keep_alive_interval(indexer_grpc_http2_ping_interval) - .keep_alive_timeout(indexer_grpc_http2_ping_timeout); + .http2_keep_alive_interval(transaction_stream_config.indexer_grpc_http2_ping_interval()) + .keep_alive_timeout(transaction_stream_config.indexer_grpc_http2_ping_timeout()); // If the scheme is https, add a TLS config. - let channel = if indexer_grpc_data_service_address.scheme() == "https" { + let channel = if transaction_stream_config + .indexer_grpc_data_service_address + .scheme() + == "https" + { let config = tonic::transport::channel::ClientTlsConfig::new(); channel .tls_config(config) @@ -114,9 +114,11 @@ pub async fn get_stream( info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - end_version = ending_version, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, "[Parser] Setting up GRPC client" ); @@ -125,7 +127,7 @@ pub async fn get_stream( let mut connect_retries = 0; let connect_res = loop { let res = timeout( - indexer_grpc_reconnection_timeout_secs, + transaction_stream_config.indexer_grpc_reconnection_timeout(), RawDataClient::connect(channel.clone()), ) .await; @@ -135,9 +137,9 @@ pub async fn get_stream( error!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - end_version = ending_version, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, retries = connect_retries, error = ?e, "[Parser] Error connecting to GRPC client" @@ -162,22 +164,24 @@ pub async fn get_stream( error!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - ending_version = ending_version, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + start_version = transaction_stream_config.starting_version, + ending_version = transaction_stream_config.request_ending_version, error = ?e, "[Parser] Error connecting to GRPC client" ); panic!("[Parser] Error connecting to GRPC client"); }, }; - let count = ending_version.map(|v| (v as i64 - starting_version as i64 + 1) as u64); + let count = transaction_stream_config + .request_ending_version + .map(|v| (v as i64 - transaction_stream_config.starting_version as i64 + 1) as u64); info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - end_version = ending_version, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, num_of_transactions = ?count, "[Parser] Setting up GRPC stream", ); @@ -186,15 +190,18 @@ pub async fn get_stream( // Retry this connection a few times before giving up let mut connect_retries = 0; let stream_res = loop { - let timeout_res = timeout(indexer_grpc_reconnection_timeout_secs, async { - let request = grpc_request_builder( - starting_version, - count, - auth_token.clone(), - processor_name.clone(), - ); - rpc_client.get_transactions(request).await - }) + let timeout_res = timeout( + transaction_stream_config.indexer_grpc_reconnection_timeout(), + async { + let request = grpc_request_builder( + transaction_stream_config.starting_version, + count, + transaction_stream_config.auth_token.clone(), + processor_name.clone(), + ); + rpc_client.get_transactions(request).await + }, + ) .await; match timeout_res { Ok(client) => break Ok(client), @@ -202,9 +209,9 @@ pub async fn get_stream( error!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - end_version = ending_version, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, retries = connect_retries, error = ?e, "[Parser] Timeout making grpc request. Retrying...", @@ -224,9 +231,9 @@ pub async fn get_stream( error!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version = starting_version, - ending_version = ending_version, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + start_version = transaction_stream_config.starting_version, + ending_version = transaction_stream_config.request_ending_version, error = ?e, "[Parser] Failed to get grpc response. Is the server running?" ); @@ -236,27 +243,25 @@ pub async fn get_stream( } pub async fn get_chain_id( - indexer_grpc_data_service_address: Url, - indexer_grpc_http2_ping_interval: Duration, - indexer_grpc_http2_ping_timeout: Duration, - indexer_grpc_reconnection_timeout_secs: Duration, - auth_token: String, + transaction_stream_config: TransactionStreamConfig, processor_name: String, ) -> u64 { info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), "[Parser] Connecting to GRPC stream to get chain id", ); + + let transaction_stream_config_for_chain_id = TransactionStreamConfig { + starting_version: 1, + request_ending_version: Some(2), + ..transaction_stream_config.clone() + }; let response = get_stream( - indexer_grpc_data_service_address.clone(), - indexer_grpc_http2_ping_interval, - indexer_grpc_http2_ping_timeout, - indexer_grpc_reconnection_timeout_secs, - 1, - Some(2), - auth_token.clone(), + transaction_stream_config_for_chain_id, processor_name.to_string(), ) .await; @@ -268,7 +273,9 @@ pub async fn get_chain_id( info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), connection_id, "[Parser] Successfully connected to GRPC stream to get chain id", ); @@ -279,7 +286,7 @@ pub async fn get_chain_id( error!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), connection_id, error = ?rpc_error, "[Parser] Error receiving datastream response for chain id" @@ -290,7 +297,9 @@ pub async fn get_chain_id( error!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), connection_id, "[Parser] Stream ended before getting response fo for chain id" ); @@ -300,14 +309,7 @@ pub async fn get_chain_id( } pub struct TransactionStream { - indexer_grpc_data_service_address: Url, - indexer_grpc_http2_ping_interval: Duration, - indexer_grpc_http2_ping_timeout: Duration, - indexer_grpc_reconnection_timeout_secs: Duration, - indexer_grpc_response_item_timeout_secs: Duration, - starting_version: u64, - request_ending_version: Option, - auth_token: String, + transaction_stream_config: TransactionStreamConfig, processor_name: String, transaction_filter: TransactionFilter, pb_channel_txn_chunk_size: usize, @@ -325,57 +327,43 @@ pub struct TransactionStreamOutput { } impl TransactionStream { - pub fn new( - indexer_grpc_data_service_address: Url, - indexer_grpc_http2_ping_interval: Duration, - indexer_grpc_http2_ping_timeout: Duration, - indexer_grpc_reconnection_timeout_secs: Duration, - indexer_grpc_response_item_timeout_secs: Duration, - starting_version: u64, - request_ending_version: Option, - auth_token: String, + pub async fn new( + transaction_stream_config: TransactionStreamConfig, processor_name: String, transaction_filter: TransactionFilter, pb_channel_txn_chunk_size: usize, - ) -> Self { - Self { - indexer_grpc_data_service_address, - indexer_grpc_http2_ping_interval, - indexer_grpc_http2_ping_timeout, - indexer_grpc_reconnection_timeout_secs, - indexer_grpc_response_item_timeout_secs, - starting_version, - request_ending_version, - auth_token, + ) -> Result { + let mut transaction_stream = Self { + transaction_stream_config: transaction_stream_config.clone(), processor_name, transaction_filter, pb_channel_txn_chunk_size, resp_stream: None, connection_id: None, - next_version_to_fetch: starting_version, + next_version_to_fetch: transaction_stream_config.starting_version, reconnection_retries: 0, - last_fetched_version: starting_version as i64 - 1, + last_fetched_version: transaction_stream_config.starting_version as i64 - 1, fetch_ma: MovingAverage::new(3000), - } + }; + + transaction_stream.init_stream().await; + Ok(transaction_stream) } - pub async fn init_stream(&mut self) { + async fn init_stream(&mut self) { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), - start_version = self.starting_version, - end_version = self.request_ending_version, + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + start_version = self.transaction_stream_config.starting_version, + end_version = self.transaction_stream_config.request_ending_version, "[Parser] Connecting to GRPC stream", ); let response = get_stream( - self.indexer_grpc_data_service_address.clone(), - self.indexer_grpc_http2_ping_interval, - self.indexer_grpc_http2_ping_timeout, - self.indexer_grpc_reconnection_timeout_secs, - self.starting_version, - self.request_ending_version, - self.auth_token.clone(), + self.transaction_stream_config.clone(), self.processor_name.to_string(), ) .await; @@ -388,10 +376,13 @@ impl TransactionStream { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), connection_id = self.connection_id, - start_version = self.starting_version, - end_version = self.request_ending_version, + start_version = self.transaction_stream_config.starting_version, + end_version = self.transaction_stream_config.request_ending_version, "[Parser] Successfully connected to GRPC stream", ); } @@ -411,7 +402,8 @@ impl TransactionStream { let mut transaction_pb_response = vec![]; let is_success = match tokio::time::timeout( - self.indexer_grpc_response_item_timeout_secs, + self.transaction_stream_config + .indexer_grpc_response_item_timeout(), self.resp_stream .as_mut() .expect("[Parser] GRPC stream is not initialized. Did you call init_stream?") @@ -451,7 +443,10 @@ impl TransactionStream { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), connection_id = self.connection_id, start_version, end_version, @@ -553,10 +548,10 @@ impl TransactionStream { tracing::warn!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self.transaction_stream_config.indexer_grpc_data_service_address.to_string(), self.connection_id, - start_version = self.starting_version, - end_version = self.request_ending_version, + start_version = self.transaction_stream_config.starting_version, + end_version = self.transaction_stream_config.request_ending_version, error = ?rpc_error, "[Parser] Error receiving datastream response." ); @@ -567,10 +562,13 @@ impl TransactionStream { tracing::warn!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), connection_id = self.connection_id, - start_version = self.starting_version, - end_version = self.request_ending_version, + start_version = self.transaction_stream_config.starting_version, + end_version = self.transaction_stream_config.request_ending_version, "[Parser] Stream ended." ); false @@ -582,10 +580,10 @@ impl TransactionStream { tracing::warn!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self.transaction_stream_config.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, - start_version = self.starting_version, - end_version = self.request_ending_version, + start_version = self.transaction_stream_config.starting_version, + end_version = self.transaction_stream_config.request_ending_version, error = ?e, "[Parser] Timeout receiving datastream response." ); @@ -593,19 +591,23 @@ impl TransactionStream { }, }; // Check if we're at the end of the stream - let is_end = if let Some(ending_version) = self.request_ending_version { - self.next_version_to_fetch > ending_version - } else { - false - }; + let is_end = + if let Some(ending_version) = self.transaction_stream_config.request_ending_version { + self.next_version_to_fetch > ending_version + } else { + false + }; if is_end { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), connection_id = self.connection_id, - ending_version = self.request_ending_version, + ending_version = self.transaction_stream_config.request_ending_version, next_version_to_fetch = self.next_version_to_fetch, "[Parser] Reached ending version.", ); @@ -636,7 +638,10 @@ impl TransactionStream { error!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), "[Parser] Reconnected more than 100 times. Will not retry.", ); panic!("[Parser] Reconnected more than 100 times. Will not retry.") @@ -645,20 +650,17 @@ impl TransactionStream { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), starting_version = self.next_version_to_fetch, - ending_version = self.request_ending_version, + ending_version = self.transaction_stream_config.request_ending_version, reconnection_retries = self.reconnection_retries, "[Parser] Reconnecting to GRPC stream" ); let response = get_stream( - self.indexer_grpc_data_service_address.clone(), - self.indexer_grpc_http2_ping_interval, - self.indexer_grpc_http2_ping_timeout, - self.indexer_grpc_reconnection_timeout_secs, - self.next_version_to_fetch, - self.request_ending_version, - self.auth_token.clone(), + self.transaction_stream_config.clone(), self.processor_name.to_string(), ) .await; @@ -671,148 +673,19 @@ impl TransactionStream { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self.indexer_grpc_data_service_address.to_string(), + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), connection_id = self.connection_id, starting_version = self.next_version_to_fetch, - ending_version = self.request_ending_version, + ending_version = self.transaction_stream_config.request_ending_version, reconnection_retries = self.reconnection_retries, "[Parser] Successfully reconnected to GRPC stream" ); } pub async fn get_chain_id(self) -> u64 { - get_chain_id( - self.indexer_grpc_data_service_address, - self.indexer_grpc_http2_ping_interval, - self.indexer_grpc_http2_ping_timeout, - self.indexer_grpc_reconnection_timeout_secs, - self.auth_token, - self.processor_name, - ) - .await - } -} - -pub async fn create_fetcher_loop( - txn_sender: AsyncSender, - indexer_grpc_data_service_address: Url, - indexer_grpc_http2_ping_interval: Duration, - indexer_grpc_http2_ping_timeout: Duration, - indexer_grpc_reconnection_timeout_secs: Duration, - indexer_grpc_response_item_timeout_secs: Duration, - starting_version: u64, - request_ending_version: Option, - auth_token: String, - processor_name: String, - transaction_filter: TransactionFilter, - // The number of transactions per protobuf batch - pb_channel_txn_chunk_size: usize, -) { - let mut transaction_stream = TransactionStream::new( - indexer_grpc_data_service_address.clone(), - indexer_grpc_http2_ping_interval, - indexer_grpc_http2_ping_timeout, - indexer_grpc_reconnection_timeout_secs, - indexer_grpc_response_item_timeout_secs, - starting_version, - request_ending_version, - auth_token, - processor_name.clone(), - transaction_filter, - pb_channel_txn_chunk_size, - ); - - transaction_stream.init_stream().await; - let mut send_ma = MovingAverage::new(3000); - - loop { - let transactions_output = transaction_stream.get_next_transaction_batch().await; - let start_version = transactions_output - .transactions - .first() - .unwrap() - .transactions - .first() - .unwrap() - .version; - let end_version = transactions_output - .transactions - .last() - .unwrap() - .transactions - .last() - .unwrap() - .version; - let size_in_bytes = transactions_output - .transactions - .iter() - .map(|txn_pb| txn_pb.size_in_bytes) - .sum::(); - - let txn_channel_send_latency = std::time::Instant::now(); - for txn_pb in transactions_output.transactions { - match txn_sender.send(txn_pb).await { - Ok(()) => {}, - Err(e) => { - error!( - processor_name = processor_name, - stream_address = indexer_grpc_data_service_address.to_string(), - error = ?e, - "[Parser] Error sending GRPC response to channel." - ); - panic!("[Parser] Error sending GRPC response to channel.") - }, - } - } - - // Log channel send metrics - let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); - let num_txns = end_version - start_version + 1; - - send_ma.tick_now(num_txns as u64); - let tps = send_ma.avg().ceil() as u64; - let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; - let channel_size = txn_sender.len(); - debug!( - processor_name.service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - start_version, - end_version, - channel_size, - size_in_bytes, - duration_in_secs, - bytes_per_sec, - tps, - "[Parser] Successfully sent transactions to channel." - ); - FETCHER_THREAD_CHANNEL_SIZE - .with_label_values(&[&processor_name]) - .set(channel_size as i64); - - // Handle reaching end of stream - if !transactions_output.should_continue_fetching { - // Wait for the fetched transactions to finish processing before closing the channel - loop { - let channel_size = txn_sender.len(); - info!( - processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - channel_size, - "[Parser] Waiting for channel to be empty" - ); - if channel_size.is_zero() { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - info!( - processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = indexer_grpc_data_service_address.to_string(), - "[Parser] Transaction fetcher send channel is closed." - ); - break; - } + get_chain_id(self.transaction_stream_config, self.processor_name).await } } diff --git a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs index ac7fecaac..6323b8394 100644 --- a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs +++ b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs @@ -39,6 +39,8 @@ impl ProcessorStep { } } +pub const TRANSACTION_STREAM_METRICS_PREFIX: &str = "transaction_stream_"; + /// These metrics are temporary (suffixed with _temp to avoid conflict with metrics in processor crate) /// They're only defined in this crate for backwards compatibility before we migrate over to /// using the instrumentation provided by SDK @@ -46,7 +48,7 @@ impl ProcessorStep { /// Max version processed pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { register_int_gauge_vec!( - "indexer_processor_latest_version_temp", + format!("{}_latest_version", TRANSACTION_STREAM_METRICS_PREFIX), "Latest version a processor has fully consumed", &["processor_name", "step", "message", "task_index"] ) @@ -56,7 +58,10 @@ pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { /// Count of bytes processed. pub static PROCESSED_BYTES_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( - "indexer_processor_processed_bytes_count_temp", + format!( + "{}_processed_bytes_count", + TRANSACTION_STREAM_METRICS_PREFIX + ), "Count of bytes processed", &["processor_name", "step", "message", "task_index"] ) @@ -66,7 +71,10 @@ pub static PROCESSED_BYTES_COUNT: Lazy = Lazy::new(|| { /// Count of transactions processed. pub static NUM_TRANSACTIONS_PROCESSED_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( - "indexer_processor_num_transactions_processed_count_temp", + format!( + "{}_num_transactions_processed_count", + TRANSACTION_STREAM_METRICS_PREFIX + ), "Number of transactions processed", &["processor_name", "step", "message", "task_index"] ) @@ -76,7 +84,10 @@ pub static NUM_TRANSACTIONS_PROCESSED_COUNT: Lazy = Lazy::new(|| /// Transaction timestamp in unixtime pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { register_gauge_vec!( - "indexer_processor_transaction_unix_timestamp_temp", + format!( + "{}_transaction_unix_timestamp", + TRANSACTION_STREAM_METRICS_PREFIX + ), "Transaction timestamp in unixtime", &["processor_name", "step", "message", "task_index"] ) @@ -86,19 +97,12 @@ pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { /// Count of transactions filtered out pub static NUM_TRANSACTIONS_FILTERED_OUT_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( - "indexer_processor_num_transactions_filtered_out_count", + format!( + "{}_num_transactions_filtered_out_count", + TRANSACTION_STREAM_METRICS_PREFIX + ), "Number of transactions filtered out", &["processor_name"] ) .unwrap() }); - -/// Size of the channel containing transactions fetched from GRPC, waiting to be processed -pub static FETCHER_THREAD_CHANNEL_SIZE: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "indexer_processor_fetcher_thread_channel_size", - "Size of the fetcher thread channel", - &["processor_name"] - ) - .unwrap() -}); diff --git a/rust/processor/src/config.rs b/rust/processor/src/config.rs index 2f2a47fc5..4d9aad452 100644 --- a/rust/processor/src/config.rs +++ b/rust/processor/src/config.rs @@ -6,9 +6,9 @@ use crate::{ }; use ahash::AHashMap; use anyhow::{Context, Result}; +use aptos_indexer_transaction_stream::TransactionStreamConfig; use serde::{Deserialize, Serialize}; use server_framework::RunnableConfig; -use std::time::Duration; use transaction_filter::transaction_filter::TransactionFilter; use url::Url; @@ -44,7 +44,7 @@ pub struct IndexerGrpcProcessorConfig { pub per_table_chunk_sizes: AHashMap, pub enable_verbose_logging: Option, - #[serde(default = "IndexerGrpcProcessorConfig::default_grpc_response_item_timeout_in_secs")] + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_response_item_timeout")] pub grpc_response_item_timeout_in_secs: u64, #[serde(default)] @@ -69,11 +69,6 @@ impl IndexerGrpcProcessorConfig { pub const fn default_pb_channel_txn_chunk_size() -> usize { 100_000 } - - /// Default timeout for grpc response item in seconds. Defaults to 60 seconds. - pub const fn default_grpc_response_item_timeout_in_secs() -> u64 { - 60 - } } #[async_trait::async_trait] @@ -116,39 +111,17 @@ impl RunnableConfig for IndexerGrpcProcessorConfig { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -#[serde(default)] pub struct IndexerGrpcHttp2Config { /// Indexer GRPC http2 ping interval in seconds. Defaults to 30. /// Tonic ref: https://docs.rs/tonic/latest/tonic/transport/channel/struct.Endpoint.html#method.http2_keep_alive_interval - indexer_grpc_http2_ping_interval_in_secs: u64, + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_http2_ping_interval")] + pub indexer_grpc_http2_ping_interval_in_secs: u64, /// Indexer GRPC http2 ping timeout in seconds. Defaults to 10. - indexer_grpc_http2_ping_timeout_in_secs: u64, + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_http2_ping_timeout")] + pub indexer_grpc_http2_ping_timeout_in_secs: u64, /// Seconds before timeout for grpc connection. - indexer_grpc_connection_timeout_secs: u64, -} - -impl IndexerGrpcHttp2Config { - pub fn grpc_http2_ping_interval_in_secs(&self) -> Duration { - Duration::from_secs(self.indexer_grpc_http2_ping_interval_in_secs) - } - - pub fn grpc_http2_ping_timeout_in_secs(&self) -> Duration { - Duration::from_secs(self.indexer_grpc_http2_ping_timeout_in_secs) - } - - pub fn grpc_connection_timeout_secs(&self) -> Duration { - Duration::from_secs(self.indexer_grpc_connection_timeout_secs) - } -} - -impl Default for IndexerGrpcHttp2Config { - fn default() -> Self { - Self { - indexer_grpc_http2_ping_interval_in_secs: 30, - indexer_grpc_http2_ping_timeout_in_secs: 10, - indexer_grpc_connection_timeout_secs: 5, - } - } + #[serde(default = "TransactionStreamConfig::default_indexer_grpc_reconnection_timeout")] + pub indexer_grpc_connection_timeout_secs: u64, } diff --git a/rust/processor/src/utils/counters.rs b/rust/processor/src/utils/counters.rs index f1fed7266..cab11e05d 100644 --- a/rust/processor/src/utils/counters.rs +++ b/rust/processor/src/utils/counters.rs @@ -236,3 +236,13 @@ pub static PROCESSOR_UNKNOWN_TYPE_COUNT: Lazy = Lazy::new(|| { ) .unwrap() }); + +/// Size of the channel containing transactions fetched from GRPC, waiting to be processed +pub static FETCHER_THREAD_CHANNEL_SIZE: Lazy = Lazy::new(|| { + register_int_gauge_vec!( + "indexer_processor_fetcher_thread_channel_size", + "Size of the fetcher thread channel", + &["processor_name"] + ) + .unwrap() +}); diff --git a/rust/processor/src/worker.rs b/rust/processor/src/worker.rs index dbf3f8c78..11d8cca25 100644 --- a/rust/processor/src/worker.rs +++ b/rust/processor/src/worker.rs @@ -18,11 +18,11 @@ use crate::{ schema::ledger_infos, utils::{ counters::{ - ProcessorStep, GRPC_LATENCY_BY_PROCESSOR_IN_SECS, LATEST_PROCESSED_VERSION, - NUM_TRANSACTIONS_PROCESSED_COUNT, PB_CHANNEL_FETCH_WAIT_TIME_SECS, - PROCESSED_BYTES_COUNT, PROCESSOR_DATA_PROCESSED_LATENCY_IN_SECS, - PROCESSOR_DATA_RECEIVED_LATENCY_IN_SECS, PROCESSOR_ERRORS_COUNT, - PROCESSOR_INVOCATIONS_COUNT, PROCESSOR_SUCCESSES_COUNT, + ProcessorStep, FETCHER_THREAD_CHANNEL_SIZE, GRPC_LATENCY_BY_PROCESSOR_IN_SECS, + LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_PROCESSED_COUNT, + PB_CHANNEL_FETCH_WAIT_TIME_SECS, PROCESSED_BYTES_COUNT, + PROCESSOR_DATA_PROCESSED_LATENCY_IN_SECS, PROCESSOR_DATA_RECEIVED_LATENCY_IN_SECS, + PROCESSOR_ERRORS_COUNT, PROCESSOR_INVOCATIONS_COUNT, PROCESSOR_SUCCESSES_COUNT, SINGLE_BATCH_DB_INSERTION_TIME_IN_SECS, SINGLE_BATCH_PARSING_TIME_IN_SECS, SINGLE_BATCH_PROCESSING_TIME_IN_SECS, TRANSACTION_UNIX_TIMESTAMP, }, @@ -32,8 +32,13 @@ use crate::{ }; use ahash::AHashMap; use anyhow::{Context, Result}; -use aptos_indexer_transaction_stream::transaction_stream::TransactionsPBResponse; +use aptos_indexer_transaction_stream::{ + config::TransactionStreamConfig, TransactionStream, TransactionsPBResponse, +}; use aptos_moving_average::MovingAverage; +use bigdecimal::Zero; +use kanal::AsyncSender; +use std::time::Duration; use tokio::task::JoinHandle; use tracing::{debug, error, info}; use transaction_filter::transaction_filter::TransactionFilter; @@ -169,14 +174,26 @@ impl Worker { ); let concurrent_tasks = self.number_concurrent_processing_tasks; + let transaction_stream_config = TransactionStreamConfig { + indexer_grpc_data_service_address: self.indexer_grpc_data_service_address.clone(), + starting_version, + request_ending_version: self.ending_version, + auth_token: self.auth_token.clone(), + indexer_grpc_http2_ping_interval_secs: self + .grpc_http2_config + .indexer_grpc_http2_ping_interval_in_secs, + indexer_grpc_http2_ping_timeout_secs: self + .grpc_http2_config + .indexer_grpc_http2_ping_timeout_in_secs, + indexer_grpc_reconnection_timeout_secs: self + .grpc_http2_config + .indexer_grpc_connection_timeout_secs, + indexer_grpc_response_item_timeout_secs: self.grpc_response_item_timeout_in_secs, + }; // get the chain id let chain_id = aptos_indexer_transaction_stream::transaction_stream::get_chain_id( - self.indexer_grpc_data_service_address.clone(), - self.grpc_http2_config.grpc_http2_ping_interval_in_secs(), - self.grpc_http2_config.grpc_http2_ping_timeout_in_secs(), - self.grpc_http2_config.grpc_connection_timeout_secs(), - self.auth_token.clone(), + transaction_stream_config.clone(), processor_name.to_string(), ) .await; @@ -187,24 +204,13 @@ impl Worker { self.grpc_chain_id = Some(chain_id); let ending_version = self.ending_version; - let indexer_grpc_data_service_address = self.indexer_grpc_data_service_address.clone(); - let indexer_grpc_http2_ping_interval = - self.grpc_http2_config.grpc_http2_ping_interval_in_secs(); - let indexer_grpc_http2_ping_timeout = - self.grpc_http2_config.grpc_http2_ping_timeout_in_secs(); - let indexer_grpc_reconnection_timeout_secs = - self.grpc_http2_config.grpc_connection_timeout_secs(); - let pb_channel_txn_chunk_size = self.pb_channel_txn_chunk_size; // Create a transaction fetcher thread that will continuously fetch transactions from the GRPC stream // and write into a channel // TODO: change channel size based on number_concurrent_processing_tasks let (tx, receiver) = kanal::bounded_async::(BUFFER_SIZE); - let request_ending_version = self.ending_version; - let auth_token = self.auth_token.clone(); let transaction_filter = self.transaction_filter.clone(); - let grpc_response_item_timeout = - std::time::Duration::from_secs(self.grpc_response_item_timeout_in_secs); + let pb_channel_txn_chunk_size = self.pb_channel_txn_chunk_size; let fetcher_task = tokio::spawn(async move { info!( processor_name = processor_name, @@ -214,16 +220,9 @@ impl Worker { "[Parser] Starting fetcher thread" ); - aptos_indexer_transaction_stream::transaction_stream::create_fetcher_loop( + fetch_transactions_from_transaction_stream( tx.clone(), - indexer_grpc_data_service_address.clone(), - indexer_grpc_http2_ping_interval, - indexer_grpc_http2_ping_timeout, - indexer_grpc_reconnection_timeout_secs, - grpc_response_item_timeout, - starting_version, - request_ending_version, - auth_token.clone(), + transaction_stream_config.clone(), processor_name.to_string(), transaction_filter, pb_channel_txn_chunk_size, @@ -616,6 +615,135 @@ impl Worker { } } +pub async fn fetch_transactions_from_transaction_stream( + txn_sender: AsyncSender, + transaction_stream_config: TransactionStreamConfig, + processor_name: String, + transaction_filter: TransactionFilter, + // The number of transactions per protobuf batch + pb_channel_txn_chunk_size: usize, +) { + let transaction_stream_res = TransactionStream::new( + transaction_stream_config.clone(), + processor_name.clone(), + transaction_filter, + pb_channel_txn_chunk_size, + ) + .await; + + let mut transaction_stream = match transaction_stream_res { + Ok(stream) => stream, + Err(e) => { + error!( + processor_name = processor_name, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + error = ?e, + "[Parser] Error creating transaction stream." + ); + panic!("[Parser] Error creating transaction stream.") + }, + }; + + let mut send_ma = MovingAverage::new(3000); + + loop { + let transactions_output = transaction_stream.get_next_transaction_batch().await; + let start_version = transactions_output + .transactions + .first() + .unwrap() + .transactions + .first() + .unwrap() + .version; + let end_version = transactions_output + .transactions + .last() + .unwrap() + .transactions + .last() + .unwrap() + .version; + let size_in_bytes = transactions_output + .transactions + .iter() + .map(|txn_pb| txn_pb.size_in_bytes) + .sum::(); + + let txn_channel_send_latency = std::time::Instant::now(); + for txn_pb in transactions_output.transactions { + match txn_sender.send(txn_pb).await { + Ok(()) => {}, + Err(e) => { + error!( + processor_name = processor_name, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + error = ?e, + "[Parser] Error sending GRPC response to channel." + ); + panic!("[Parser] Error sending GRPC response to channel.") + }, + } + } + + // Log channel send metrics + let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); + let num_txns = end_version - start_version + 1; + + send_ma.tick_now(num_txns as u64); + let tps = send_ma.avg().ceil() as u64; + let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; + let channel_size = txn_sender.len(); + debug!( + processor_name.service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + start_version, + end_version, + channel_size, + size_in_bytes, + duration_in_secs, + bytes_per_sec, + tps, + "[Parser] Successfully sent transactions to channel." + ); + FETCHER_THREAD_CHANNEL_SIZE + .with_label_values(&[&processor_name]) + .set(channel_size as i64); + + // Handle reaching end of stream + if !transactions_output.should_continue_fetching { + // Wait for the fetched transactions to finish processing before closing the channel + loop { + let channel_size = txn_sender.len(); + info!( + processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + channel_size, + "[Parser] Waiting for channel to be empty" + ); + if channel_size.is_zero() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + info!( + processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + "[Parser] Transaction fetcher send channel is closed." + ); + break; + } + } +} + async fn fetch_transactions( processor_name: &str, stream_address: &str, From 63fbfb70fed665f7ee774b9eb69080998e5f7bca Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:08:10 -0400 Subject: [PATCH 08/13] Move chunking & filtering to processor crate --- .../src/transaction_stream.rs | 132 +++------- .../src/utils/counters.rs | 13 - rust/processor/src/utils/counters.rs | 10 + rust/processor/src/worker.rs | 226 ++++++++++-------- 4 files changed, 175 insertions(+), 206 deletions(-) diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 57a5320b0..c8d387c63 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -1,12 +1,12 @@ use crate::config::TransactionStreamConfig; use crate::utils::{ counters::{ - ProcessorStep, LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_FILTERED_OUT_COUNT, - NUM_TRANSACTIONS_PROCESSED_COUNT, PROCESSED_BYTES_COUNT, TRANSACTION_UNIX_TIMESTAMP, + ProcessorStep, LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_PROCESSED_COUNT, + PROCESSED_BYTES_COUNT, TRANSACTION_UNIX_TIMESTAMP, }, util::{timestamp_to_iso, timestamp_to_unixtime}, }; -use anyhow::Result; +use anyhow::{anyhow, Result}; use aptos_moving_average::MovingAverage; use aptos_protos::{ indexer::v1::{raw_data_client::RawDataClient, GetTransactionsRequest, TransactionsResponse}, @@ -20,7 +20,6 @@ use std::time::Duration; use tokio::time::timeout; use tonic::{Response, Streaming}; use tracing::{error, info}; -use transaction_filter::transaction_filter::TransactionFilter; /// GRPC request metadata key for the token ID. const GRPC_API_GATEWAY_API_KEY_HEADER: &str = "authorization"; @@ -311,8 +310,6 @@ pub async fn get_chain_id( pub struct TransactionStream { transaction_stream_config: TransactionStreamConfig, processor_name: String, - transaction_filter: TransactionFilter, - pb_channel_txn_chunk_size: usize, resp_stream: Option>, connection_id: Option, next_version_to_fetch: u64, @@ -322,7 +319,7 @@ pub struct TransactionStream { } pub struct TransactionStreamOutput { - pub transactions: Vec, + pub transactions: TransactionsPBResponse, pub should_continue_fetching: bool, } @@ -330,14 +327,10 @@ impl TransactionStream { pub async fn new( transaction_stream_config: TransactionStreamConfig, processor_name: String, - transaction_filter: TransactionFilter, - pb_channel_txn_chunk_size: usize, ) -> Result { let mut transaction_stream = Self { transaction_stream_config: transaction_stream_config.clone(), processor_name, - transaction_filter, - pb_channel_txn_chunk_size, resp_stream: None, connection_id: None, next_version_to_fetch: transaction_stream_config.starting_version, @@ -397,11 +390,12 @@ impl TransactionStream { /// Returns /// - true if should continue fetching /// - false if we reached the end of the stream or there is an error and the loop should stop - pub async fn get_next_transaction_batch(&mut self) -> TransactionStreamOutput { + pub async fn get_next_transaction_batch_with_reconnect( + &mut self, + ) -> Result { let grpc_channel_recv_latency = std::time::Instant::now(); - let mut transaction_pb_response = vec![]; - let is_success = match tokio::time::timeout( + let txn_pb_res = match tokio::time::timeout( self.transaction_stream_config .indexer_grpc_response_item_timeout(), self.resp_stream @@ -432,11 +426,11 @@ impl TransactionStream { self.fetch_ma.tick_now(num_txns as u64); // Filter out the txns we don't care about - r.transactions - .retain(|txn| self.transaction_filter.include(txn)); + // r.transactions + // .retain(|txn| self.transaction_filter.include(txn)); - let num_txn_post_filter = r.transactions.len(); - let num_filtered_txns = num_txns - num_txn_post_filter; + // let num_txn_post_filter = r.transactions.len(); + // let num_filtered_txns = num_txns - num_txn_post_filter; let step = ProcessorStep::ReceivedTxnsFromGrpc.get_step(); let label = ProcessorStep::ReceivedTxnsFromGrpc.get_label(); @@ -459,7 +453,7 @@ impl TransactionStream { .map(timestamp_to_iso) .unwrap_or_default(), num_of_transactions = end_version - start_version + 1, - num_filtered_txns, + // num_filtered_txns, size_in_bytes, duration_in_secs, tps = self.fetch_ma.avg().ceil() as u64, @@ -498,50 +492,17 @@ impl TransactionStream { .with_label_values(&[&self.processor_name, step, label, "-"]) .inc_by(end_version - start_version + 1); - //potentially break txn_pb into many `TransactionsPBResponse` that are each `pb_channel_txn_chunk_size` txns max in size - if num_txn_post_filter < self.pb_channel_txn_chunk_size { - // We only need to send one; avoid the chunk/clone - let txn_pb = TransactionsPBResponse { - transactions: r.transactions, - chain_id, - start_version, - end_version, - start_txn_timestamp, - end_txn_timestamp, - size_in_bytes, - }; - transaction_pb_response.push(txn_pb); - } else { - // We are breaking down a big batch into small batches; this involves an iterator - let average_size_in_bytes = size_in_bytes / num_txns as u64; - - let pb_txn_chunks: Vec> = r - .transactions - .into_iter() - .chunks(self.pb_channel_txn_chunk_size) - .into_iter() - .map(|chunk| chunk.collect()) - .collect(); - for txns in pb_txn_chunks { - let size_in_bytes = average_size_in_bytes * txns.len() as u64; - let txn_pb = TransactionsPBResponse { - transactions: txns, - chain_id, - start_version, - end_version, - // TODO: this is only for gap checker + filtered txns, but this is wrong - start_txn_timestamp: start_txn_timestamp.clone(), - end_txn_timestamp: end_txn_timestamp.clone(), - size_in_bytes, - }; - transaction_pb_response.push(txn_pb); - } - } + let txn_pb = TransactionsPBResponse { + transactions: r.transactions, + chain_id, + start_version, + end_version, + start_txn_timestamp, + end_txn_timestamp, + size_in_bytes, + }; - NUM_TRANSACTIONS_FILTERED_OUT_COUNT - .with_label_values(&[&self.processor_name]) - .inc_by(num_filtered_txns as u64); - true + Ok(txn_pb) }, // Error receiving datastream response Some(Err(rpc_error)) => { @@ -555,7 +516,7 @@ impl TransactionStream { error = ?rpc_error, "[Parser] Error receiving datastream response." ); - false + Err(anyhow!("Error receiving datastream response")) }, // Stream is finished None => { @@ -571,7 +532,7 @@ impl TransactionStream { end_version = self.transaction_stream_config.request_ending_version, "[Parser] Stream ended." ); - false + Err(anyhow!("Stream ended")) }, } }, @@ -587,45 +548,18 @@ impl TransactionStream { error = ?e, "[Parser] Timeout receiving datastream response." ); - false + Err(anyhow!("Timeout receiving datastream response")) }, }; - // Check if we're at the end of the stream - let is_end = - if let Some(ending_version) = self.transaction_stream_config.request_ending_version { - self.next_version_to_fetch > ending_version - } else { - false - }; - - if is_end { - info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self - .transaction_stream_config - .indexer_grpc_data_service_address - .to_string(), - connection_id = self.connection_id, - ending_version = self.transaction_stream_config.request_ending_version, - next_version_to_fetch = self.next_version_to_fetch, - "[Parser] Reached ending version.", - ); + txn_pb_res + } - TransactionStreamOutput { - transactions: transaction_pb_response, - should_continue_fetching: false, - } + /// Helper function to signal that we've fetched all the transactions up to the ending version that was requested. + pub fn is_end_of_stream(&self) -> bool { + if let Some(ending_version) = self.transaction_stream_config.request_ending_version { + self.next_version_to_fetch > ending_version } else { - // The rest is to see if we need to reconnect - if !is_success { - self.reconnect_to_grpc().await; - } - - TransactionStreamOutput { - transactions: transaction_pb_response, - should_continue_fetching: true, - } + false } } diff --git a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs index 6323b8394..959cd196b 100644 --- a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs +++ b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs @@ -93,16 +93,3 @@ pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { ) .unwrap() }); - -/// Count of transactions filtered out -pub static NUM_TRANSACTIONS_FILTERED_OUT_COUNT: Lazy = Lazy::new(|| { - register_int_counter_vec!( - format!( - "{}_num_transactions_filtered_out_count", - TRANSACTION_STREAM_METRICS_PREFIX - ), - "Number of transactions filtered out", - &["processor_name"] - ) - .unwrap() -}); diff --git a/rust/processor/src/utils/counters.rs b/rust/processor/src/utils/counters.rs index cab11e05d..023b44554 100644 --- a/rust/processor/src/utils/counters.rs +++ b/rust/processor/src/utils/counters.rs @@ -246,3 +246,13 @@ pub static FETCHER_THREAD_CHANNEL_SIZE: Lazy = Lazy::new(|| { ) .unwrap() }); + +/// Count of transactions filtered out +pub static NUM_TRANSACTIONS_FILTERED_OUT_COUNT: Lazy = Lazy::new(|| { + register_int_counter_vec!( + format!("indexer_processor_num_transactions_filtered_out_count",), + "Number of transactions filtered out", + &["processor_name"] + ) + .unwrap() +}); diff --git a/rust/processor/src/worker.rs b/rust/processor/src/worker.rs index 11d8cca25..72e8dddbe 100644 --- a/rust/processor/src/worker.rs +++ b/rust/processor/src/worker.rs @@ -19,10 +19,11 @@ use crate::{ utils::{ counters::{ ProcessorStep, FETCHER_THREAD_CHANNEL_SIZE, GRPC_LATENCY_BY_PROCESSOR_IN_SECS, - LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_PROCESSED_COUNT, - PB_CHANNEL_FETCH_WAIT_TIME_SECS, PROCESSED_BYTES_COUNT, - PROCESSOR_DATA_PROCESSED_LATENCY_IN_SECS, PROCESSOR_DATA_RECEIVED_LATENCY_IN_SECS, - PROCESSOR_ERRORS_COUNT, PROCESSOR_INVOCATIONS_COUNT, PROCESSOR_SUCCESSES_COUNT, + LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_FILTERED_OUT_COUNT, + NUM_TRANSACTIONS_PROCESSED_COUNT, PB_CHANNEL_FETCH_WAIT_TIME_SECS, + PROCESSED_BYTES_COUNT, PROCESSOR_DATA_PROCESSED_LATENCY_IN_SECS, + PROCESSOR_DATA_RECEIVED_LATENCY_IN_SECS, PROCESSOR_ERRORS_COUNT, + PROCESSOR_INVOCATIONS_COUNT, PROCESSOR_SUCCESSES_COUNT, SINGLE_BATCH_DB_INSERTION_TIME_IN_SECS, SINGLE_BATCH_PARSING_TIME_IN_SECS, SINGLE_BATCH_PROCESSING_TIME_IN_SECS, TRANSACTION_UNIX_TIMESTAMP, }, @@ -36,7 +37,9 @@ use aptos_indexer_transaction_stream::{ config::TransactionStreamConfig, TransactionStream, TransactionsPBResponse, }; use aptos_moving_average::MovingAverage; +use aptos_protos::transaction::v1::Transaction; use bigdecimal::Zero; +use itertools::Itertools; use kanal::AsyncSender; use std::time::Duration; use tokio::task::JoinHandle; @@ -623,13 +626,8 @@ pub async fn fetch_transactions_from_transaction_stream( // The number of transactions per protobuf batch pb_channel_txn_chunk_size: usize, ) { - let transaction_stream_res = TransactionStream::new( - transaction_stream_config.clone(), - processor_name.clone(), - transaction_filter, - pb_channel_txn_chunk_size, - ) - .await; + let transaction_stream_res = + TransactionStream::new(transaction_stream_config.clone(), processor_name.clone()).await; let mut transaction_stream = match transaction_stream_res { Ok(stream) => stream, @@ -647,99 +645,139 @@ pub async fn fetch_transactions_from_transaction_stream( let mut send_ma = MovingAverage::new(3000); loop { - let transactions_output = transaction_stream.get_next_transaction_batch().await; - let start_version = transactions_output - .transactions - .first() - .unwrap() - .transactions - .first() - .unwrap() - .version; - let end_version = transactions_output - .transactions - .last() - .unwrap() - .transactions - .last() - .unwrap() - .version; - let size_in_bytes = transactions_output - .transactions - .iter() - .map(|txn_pb| txn_pb.size_in_bytes) - .sum::(); - - let txn_channel_send_latency = std::time::Instant::now(); - for txn_pb in transactions_output.transactions { - match txn_sender.send(txn_pb).await { - Ok(()) => {}, - Err(e) => { - error!( - processor_name = processor_name, - stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), - error = ?e, - "[Parser] Error sending GRPC response to channel." - ); - panic!("[Parser] Error sending GRPC response to channel.") - }, - } - } + let transactions_res = transaction_stream + .get_next_transaction_batch_with_reconnect() + .await; - // Log channel send metrics - let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); - let num_txns = end_version - start_version + 1; - - send_ma.tick_now(num_txns as u64); - let tps = send_ma.avg().ceil() as u64; - let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; - let channel_size = txn_sender.len(); - debug!( - processor_name.service_type = PROCESSOR_SERVICE_TYPE, - stream_address = transaction_stream_config - .indexer_grpc_data_service_address - .to_string(), - start_version, - end_version, - channel_size, - size_in_bytes, - duration_in_secs, - bytes_per_sec, - tps, - "[Parser] Successfully sent transactions to channel." - ); - FETCHER_THREAD_CHANNEL_SIZE - .with_label_values(&[&processor_name]) - .set(channel_size as i64); + match transactions_res { + Ok(mut txn_pb) => { + let start_version = txn_pb.transactions.first().unwrap().version; + let end_version = txn_pb.transactions.last().unwrap().version; + let size_in_bytes = txn_pb.size_in_bytes; + let num_txns = end_version - start_version + 1; + + // Filter out the txns we don't care about + txn_pb + .transactions + .retain(|txn| transaction_filter.include(txn)); + let num_txn_post_filter = txn_pb.transactions.len() as u64; + let num_filtered_txns = num_txns - num_txn_post_filter; + + // potentially break txn_pb into many `TransactionsPBResponse` that are each `pb_channel_txn_chunk_size` txns max in size + let txn_channel_send_latency = std::time::Instant::now(); + if num_txn_post_filter < pb_channel_txn_chunk_size as u64 { + // We only need to send one; avoid the chunk/clone + match txn_sender.send(txn_pb).await { + Ok(()) => {}, + Err(e) => { + error!( + processor_name = processor_name, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + error = ?e, + "[Parser] Error sending GRPC response to channel." + ); + panic!("[Parser] Error sending GRPC response to channel.") + }, + } + } else { + // We are breaking down a big batch into small batches; this involves an iterator + let average_size_in_bytes = size_in_bytes / num_filtered_txns as u64; + + let txn_pb_chunked: Vec> = txn_pb + .transactions + .into_iter() + .chunks(pb_channel_txn_chunk_size) + .into_iter() + .map(|chunk| chunk.collect()) + .collect(); + for txn_pb_chunk in txn_pb_chunked { + let size_in_bytes = average_size_in_bytes * txn_pb_chunk.len() as u64; + let txn_pb = TransactionsPBResponse { + transactions: txn_pb_chunk, + chain_id: txn_pb.chain_id, + start_version: txn_pb.start_version, + end_version: txn_pb.end_version, + // TODO: this is only for gap checker + filtered txns, but this is wrong + start_txn_timestamp: txn_pb.start_txn_timestamp.clone(), + end_txn_timestamp: txn_pb.end_txn_timestamp.clone(), + size_in_bytes, + }; + match txn_sender.send(txn_pb).await { + Ok(()) => {}, + Err(e) => { + error!( + processor_name = processor_name, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + error = ?e, + "[Parser] Error sending GRPC response to channel." + ); + panic!("[Parser] Error sending GRPC response to channel.") + }, + } + } + } - // Handle reaching end of stream - if !transactions_output.should_continue_fetching { - // Wait for the fetched transactions to finish processing before closing the channel - loop { + // Log channel send metrics + let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); + + send_ma.tick_now(num_txns as u64); + let tps = send_ma.avg().ceil() as u64; + let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; let channel_size = txn_sender.len(); - info!( - processor_name, - service_type = PROCESSOR_SERVICE_TYPE, + debug!( + processor_name.service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), + start_version, + end_version, channel_size, - "[Parser] Waiting for channel to be empty" + size_in_bytes, + duration_in_secs, + bytes_per_sec, + tps, + "[Parser] Successfully sent transactions to channel." ); - if channel_size.is_zero() { + FETCHER_THREAD_CHANNEL_SIZE + .with_label_values(&[&processor_name]) + .set(channel_size as i64); + NUM_TRANSACTIONS_FILTERED_OUT_COUNT + .with_label_values(&[&processor_name]) + .inc_by(num_filtered_txns as u64); + }, + Err(e) => { + if transaction_stream.is_end_of_stream() { + // Wait for the fetched transactions to finish processing before closing the channel + loop { + let channel_size = txn_sender.len(); + info!( + processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + channel_size, + "[Parser] Waiting for channel to be empty" + ); + if channel_size.is_zero() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + info!( + processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + "[Parser] Transaction fetcher send channel is closed." + ); break; + } else { + // If there was an error fetching transactions, try to reconnect + transaction_stream.reconnect_to_grpc().await; } - tokio::time::sleep(Duration::from_millis(100)).await; - } - info!( - processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = transaction_stream_config - .indexer_grpc_data_service_address - .to_string(), - "[Parser] Transaction fetcher send channel is closed." - ); - break; + }, } } } From 6185d59002b1cd0fef8454c825dc65cd124d6fb7 Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Fri, 14 Jun 2024 18:04:54 -0400 Subject: [PATCH 09/13] Result everywhere --- .../src/transaction_stream.rs | 136 ++++++++++-------- rust/processor/src/worker.rs | 37 ++++- 2 files changed, 115 insertions(+), 58 deletions(-) diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index c8d387c63..56b046b60 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -73,7 +73,7 @@ pub fn grpc_request_builder( pub async fn get_stream( transaction_stream_config: TransactionStreamConfig, processor_name: String, -) -> Response> { +) -> Result>> { info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, @@ -124,14 +124,31 @@ pub async fn get_stream( // TODO: move this to a config file // Retry this connection a few times before giving up let mut connect_retries = 0; - let connect_res = loop { + let res = loop { let res = timeout( transaction_stream_config.indexer_grpc_reconnection_timeout(), RawDataClient::connect(channel.clone()), ) .await; match res { - Ok(client) => break Ok(client), + Ok(connect_res) => match connect_res { + Ok(client) => break Ok(client), + Err(e) => { + error!( + processor_name = processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, + error = ?e, + "[Parser] Error connecting to GRPC client" + ); + connect_retries += 1; + if connect_retries >= RECONNECTION_MAX_RETRIES { + break Err(anyhow!("Error connecting to GRPC client").context(e)); + } + }, + }, Err(e) => { error!( processor_name = processor_name, @@ -141,40 +158,29 @@ pub async fn get_stream( end_version = transaction_stream_config.request_ending_version, retries = connect_retries, error = ?e, - "[Parser] Error connecting to GRPC client" + "[Parser] Timed out connecting to GRPC client" ); connect_retries += 1; if connect_retries >= RECONNECTION_MAX_RETRIES { - break Err(e); + break Err(anyhow!("Timed out connecting to GRPC client")); } }, } - } - .expect("[Parser] Timeout connecting to GRPC server"); - - let mut rpc_client = match connect_res { - Ok(client) => client - .accept_compressed(tonic::codec::CompressionEncoding::Gzip) - .accept_compressed(tonic::codec::CompressionEncoding::Zstd) - .send_compressed(tonic::codec::CompressionEncoding::Zstd) - .max_decoding_message_size(MAX_RESPONSE_SIZE) - .max_encoding_message_size(MAX_RESPONSE_SIZE), - Err(e) => { - error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), - start_version = transaction_stream_config.starting_version, - ending_version = transaction_stream_config.request_ending_version, - error = ?e, - "[Parser] Error connecting to GRPC client" - ); - panic!("[Parser] Error connecting to GRPC client"); - }, }; + + let raw_data_client = res?; + + let mut rpc_client = raw_data_client + .accept_compressed(tonic::codec::CompressionEncoding::Gzip) + .accept_compressed(tonic::codec::CompressionEncoding::Zstd) + .send_compressed(tonic::codec::CompressionEncoding::Zstd) + .max_decoding_message_size(MAX_RESPONSE_SIZE) + .max_encoding_message_size(MAX_RESPONSE_SIZE); + let count = transaction_stream_config .request_ending_version .map(|v| (v as i64 - transaction_stream_config.starting_version as i64 + 1) as u64); + info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, @@ -203,7 +209,24 @@ pub async fn get_stream( ) .await; match timeout_res { - Ok(client) => break Ok(client), + Ok(response_res) => match response_res { + Ok(response) => break Ok(response), + Err(e) => { + error!( + processor_name = processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, + error = ?e, + "[Parser] Error making grpc request. Retrying..." + ); + connect_retries += 1; + if connect_retries >= RECONNECTION_MAX_RETRIES { + break Err(anyhow!("Error making grpc request").context(e)); + } + }, + }, Err(e) => { error!( processor_name = processor_name, @@ -217,34 +240,19 @@ pub async fn get_stream( ); connect_retries += 1; if connect_retries >= RECONNECTION_MAX_RETRIES { - break Err(e); + break Err(anyhow!("Timeout making grpc request").context(e)); } }, } - } - .expect("[Parser] Timed out making grpc request after max retries."); + }; - match stream_res { - Ok(stream) => stream, - Err(e) => { - error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), - start_version = transaction_stream_config.starting_version, - ending_version = transaction_stream_config.request_ending_version, - error = ?e, - "[Parser] Failed to get grpc response. Is the server running?" - ); - panic!("[Parser] Failed to get grpc response. Is the server running?"); - }, - } + stream_res } pub async fn get_chain_id( transaction_stream_config: TransactionStreamConfig, processor_name: String, -) -> u64 { +) -> Result { info!( processor_name = processor_name, service_type = PROCESSOR_SERVICE_TYPE, @@ -263,7 +271,7 @@ pub async fn get_chain_id( transaction_stream_config_for_chain_id, processor_name.to_string(), ) - .await; + .await?; let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { Some(connection_id) => connection_id.to_str().unwrap().to_string(), None => "".to_string(), @@ -280,7 +288,21 @@ pub async fn get_chain_id( ); match resp_stream.next().await { - Some(Ok(r)) => r.chain_id.expect("[Parser] Chain Id doesn't exist."), + Some(Ok(r)) => match r.chain_id { + Some(chain_id) => Ok(chain_id), + None => { + error!( + processor_name = processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + connection_id, + "[Parser] Chain Id doesn't exist." + ); + Err(anyhow!("Chain Id doesn't exist")) + }, + }, Some(Err(rpc_error)) => { error!( processor_name = processor_name, @@ -290,7 +312,7 @@ pub async fn get_chain_id( error = ?rpc_error, "[Parser] Error receiving datastream response for chain id" ); - panic!("[Parser] Error receiving datastream response for chain id"); + Err(anyhow!("Error receiving datastream response for chain id").context(rpc_error)) }, None => { error!( @@ -302,7 +324,7 @@ pub async fn get_chain_id( connection_id, "[Parser] Stream ended before getting response fo for chain id" ); - panic!("[Parser] Stream ended before getting response fo for chain id"); + Err(anyhow!("Stream ended before getting response for chain id")) }, } } @@ -343,7 +365,7 @@ impl TransactionStream { Ok(transaction_stream) } - async fn init_stream(&mut self) { + async fn init_stream(&mut self) -> Result<()> { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, @@ -359,7 +381,7 @@ impl TransactionStream { self.transaction_stream_config.clone(), self.processor_name.to_string(), ) - .await; + .await?; let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { Some(connection_id) => connection_id.to_str().unwrap().to_string(), None => "".to_string(), @@ -378,6 +400,7 @@ impl TransactionStream { end_version = self.transaction_stream_config.request_ending_version, "[Parser] Successfully connected to GRPC stream", ); + Ok(()) } /// Gets a batch of transactions from the stream. Batch size is set in the grpc server. @@ -563,7 +586,7 @@ impl TransactionStream { } } - pub async fn reconnect_to_grpc(&mut self) { + pub async fn reconnect_to_grpc(&mut self) -> Result<()> { // Sleep for 100ms between reconnect tries // TODO: Turn this into exponential backoff tokio::time::sleep(Duration::from_millis(100)).await; @@ -597,7 +620,7 @@ impl TransactionStream { self.transaction_stream_config.clone(), self.processor_name.to_string(), ) - .await; + .await?; let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { Some(connection_id) => connection_id.to_str().unwrap().to_string(), None => "".to_string(), @@ -617,9 +640,10 @@ impl TransactionStream { reconnection_retries = self.reconnection_retries, "[Parser] Successfully reconnected to GRPC stream" ); + Ok(()) } - pub async fn get_chain_id(self) -> u64 { + pub async fn get_chain_id(self) -> Result { get_chain_id(self.transaction_stream_config, self.processor_name).await } } diff --git a/rust/processor/src/worker.rs b/rust/processor/src/worker.rs index 72e8dddbe..9ce354ce4 100644 --- a/rust/processor/src/worker.rs +++ b/rust/processor/src/worker.rs @@ -199,7 +199,8 @@ impl Worker { transaction_stream_config.clone(), processor_name.to_string(), ) - .await; + .await + .expect("[Parser] Error getting chain id"); self.check_or_update_chain_id(chain_id as i64) .await .unwrap(); @@ -618,6 +619,8 @@ impl Worker { } } +/// Initializes a GRPC stream and runs fetching transactions from the stream in a loop. + pub async fn fetch_transactions_from_transaction_stream( txn_sender: AsyncSender, transaction_stream_config: TransactionStreamConfig, @@ -626,6 +629,7 @@ pub async fn fetch_transactions_from_transaction_stream( // The number of transactions per protobuf batch pb_channel_txn_chunk_size: usize, ) { + // Initialize the stream let transaction_stream_res = TransactionStream::new(transaction_stream_config.clone(), processor_name.clone()).await; @@ -644,6 +648,11 @@ pub async fn fetch_transactions_from_transaction_stream( let mut send_ma = MovingAverage::new(3000); + // Fetch transactions in a loop + // There could be several special scenarios: + // 1. If we lose the connection, we will try reconnecting X times within Y seconds before crashing. + // 2. If we specified an end version and we hit that, we will stop fetching, but we will make sure that + // all existing transactions are processed. loop { let transactions_res = transaction_stream .get_next_transaction_batch_with_reconnect() @@ -775,13 +784,37 @@ pub async fn fetch_transactions_from_transaction_stream( break; } else { // If there was an error fetching transactions, try to reconnect - transaction_stream.reconnect_to_grpc().await; + match transaction_stream.reconnect_to_grpc().await { + Ok(_) => { + info!( + processor_name = processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + "[Parser] Successfully reconnected to GRPC stream." + ); + }, + Err(e) => { + error!( + processor_name = processor_name, + service_type = PROCESSOR_SERVICE_TYPE, + stream_address = transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + error = ?e, + "[Parser] Error reconnecting to GRPC stream." + ); + panic!("[Parser] Error reconnecting to GRPC stream.") + }, + } } }, } } } +/// Fetches transactions from the channel and processes them. async fn fetch_transactions( processor_name: &str, stream_address: &str, From 2c0465656586b98a2ae8f9ffb9c8cf05f2436c62 Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Fri, 14 Jun 2024 18:35:28 -0400 Subject: [PATCH 10/13] Remove processor stuff in txn stream crate --- .../src/config.rs | 1 + .../src/lib.rs | 2 +- .../src/transaction_stream.rs | 153 ++++++------------ rust/processor/src/utils/counters.rs | 8 +- rust/processor/src/worker.rs | 15 +- 5 files changed, 63 insertions(+), 116 deletions(-) diff --git a/rust/aptos-indexer-transaction-stream/src/config.rs b/rust/aptos-indexer-transaction-stream/src/config.rs index b1ae68260..f3a173742 100644 --- a/rust/aptos-indexer-transaction-stream/src/config.rs +++ b/rust/aptos-indexer-transaction-stream/src/config.rs @@ -9,6 +9,7 @@ pub struct TransactionStreamConfig { pub starting_version: u64, pub request_ending_version: Option, pub auth_token: String, + pub request_name_header: String, #[serde(default = "TransactionStreamConfig::default_indexer_grpc_http2_ping_interval")] pub indexer_grpc_http2_ping_interval_secs: u64, #[serde(default = "TransactionStreamConfig::default_indexer_grpc_http2_ping_timeout")] diff --git a/rust/aptos-indexer-transaction-stream/src/lib.rs b/rust/aptos-indexer-transaction-stream/src/lib.rs index 336759f6f..a2ac14edb 100644 --- a/rust/aptos-indexer-transaction-stream/src/lib.rs +++ b/rust/aptos-indexer-transaction-stream/src/lib.rs @@ -3,4 +3,4 @@ pub mod transaction_stream; pub mod utils; pub use config::TransactionStreamConfig; -pub use transaction_stream::{TransactionStream, TransactionStreamOutput, TransactionsPBResponse}; +pub use transaction_stream::{TransactionStream, TransactionsPBResponse}; diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 56b046b60..5886603ce 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -1,10 +1,12 @@ -use crate::config::TransactionStreamConfig; -use crate::utils::{ - counters::{ - ProcessorStep, LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_PROCESSED_COUNT, - PROCESSED_BYTES_COUNT, TRANSACTION_UNIX_TIMESTAMP, +use crate::{ + config::TransactionStreamConfig, + utils::{ + counters::{ + ProcessorStep, LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_PROCESSED_COUNT, + PROCESSED_BYTES_COUNT, TRANSACTION_UNIX_TIMESTAMP, + }, + util::{timestamp_to_iso, timestamp_to_unixtime}, }, - util::{timestamp_to_iso, timestamp_to_unixtime}, }; use anyhow::{anyhow, Result}; use aptos_moving_average::MovingAverage; @@ -14,7 +16,6 @@ use aptos_protos::{ util::timestamp::Timestamp, }; use futures_util::StreamExt; -use itertools::Itertools; use prost::Message; use std::time::Duration; use tokio::time::timeout; @@ -51,7 +52,7 @@ pub fn grpc_request_builder( starting_version: u64, transactions_count: Option, grpc_auth_token: String, - processor_name: String, + request_name_header: String, ) -> tonic::Request { let mut request = tonic::Request::new(GetTransactionsRequest { starting_version: Some(starting_version), @@ -64,19 +65,17 @@ pub fn grpc_request_builder( .parse() .unwrap(), ); - request - .metadata_mut() - .insert(GRPC_REQUEST_NAME_HEADER, processor_name.parse().unwrap()); + request.metadata_mut().insert( + GRPC_REQUEST_NAME_HEADER, + request_name_header.parse().unwrap(), + ); request } pub async fn get_stream( transaction_stream_config: TransactionStreamConfig, - processor_name: String, ) -> Result>> { info!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), @@ -111,8 +110,6 @@ pub async fn get_stream( }; info!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), @@ -135,8 +132,6 @@ pub async fn get_stream( Ok(client) => break Ok(client), Err(e) => { error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, @@ -151,8 +146,6 @@ pub async fn get_stream( }, Err(e) => { error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, @@ -182,8 +175,6 @@ pub async fn get_stream( .map(|v| (v as i64 - transaction_stream_config.starting_version as i64 + 1) as u64); info!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, @@ -194,7 +185,7 @@ pub async fn get_stream( // TODO: move this to a config file // Retry this connection a few times before giving up let mut connect_retries = 0; - let stream_res = loop { + loop { let timeout_res = timeout( transaction_stream_config.indexer_grpc_reconnection_timeout(), async { @@ -202,7 +193,7 @@ pub async fn get_stream( transaction_stream_config.starting_version, count, transaction_stream_config.auth_token.clone(), - processor_name.clone(), + transaction_stream_config.request_name_header.clone(), ); rpc_client.get_transactions(request).await }, @@ -213,8 +204,6 @@ pub async fn get_stream( Ok(response) => break Ok(response), Err(e) => { error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, @@ -229,8 +218,6 @@ pub async fn get_stream( }, Err(e) => { error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, @@ -244,18 +231,11 @@ pub async fn get_stream( } }, } - }; - - stream_res + } } -pub async fn get_chain_id( - transaction_stream_config: TransactionStreamConfig, - processor_name: String, -) -> Result { +pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> Result { info!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), @@ -267,24 +247,17 @@ pub async fn get_chain_id( request_ending_version: Some(2), ..transaction_stream_config.clone() }; - let response = get_stream( - transaction_stream_config_for_chain_id, - processor_name.to_string(), - ) - .await?; + let response = get_stream(transaction_stream_config_for_chain_id).await?; let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { Some(connection_id) => connection_id.to_str().unwrap().to_string(), None => "".to_string(), }; let mut resp_stream = response.into_inner(); info!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - connection_id, - "[Parser] Successfully connected to GRPC stream to get chain id", + connection_id, "[Parser] Successfully connected to GRPC stream to get chain id", ); match resp_stream.next().await { @@ -292,21 +265,16 @@ pub async fn get_chain_id( Some(chain_id) => Ok(chain_id), None => { error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - connection_id, - "[Parser] Chain Id doesn't exist." + connection_id, "[Parser] Chain Id doesn't exist." ); Err(anyhow!("Chain Id doesn't exist")) }, }, Some(Err(rpc_error)) => { error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), connection_id, error = ?rpc_error, @@ -316,13 +284,10 @@ pub async fn get_chain_id( }, None => { error!( - processor_name = processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - connection_id, - "[Parser] Stream ended before getting response fo for chain id" + connection_id, "[Parser] Stream ended before getting response fo for chain id" ); Err(anyhow!("Stream ended before getting response for chain id")) }, @@ -332,75 +297,58 @@ pub async fn get_chain_id( pub struct TransactionStream { transaction_stream_config: TransactionStreamConfig, processor_name: String, - resp_stream: Option>, - connection_id: Option, + stream: Streaming, + connection_id: String, next_version_to_fetch: u64, reconnection_retries: u64, last_fetched_version: i64, fetch_ma: MovingAverage, } -pub struct TransactionStreamOutput { - pub transactions: TransactionsPBResponse, - pub should_continue_fetching: bool, -} - impl TransactionStream { pub async fn new( transaction_stream_config: TransactionStreamConfig, processor_name: String, ) -> Result { - let mut transaction_stream = Self { + let (stream, connection_id) = Self::init_stream(transaction_stream_config.clone()).await?; + Ok(Self { transaction_stream_config: transaction_stream_config.clone(), processor_name, - resp_stream: None, - connection_id: None, + stream, + connection_id, next_version_to_fetch: transaction_stream_config.starting_version, reconnection_retries: 0, last_fetched_version: transaction_stream_config.starting_version as i64 - 1, fetch_ma: MovingAverage::new(3000), - }; - - transaction_stream.init_stream().await; - Ok(transaction_stream) + }) } - async fn init_stream(&mut self) -> Result<()> { + async fn init_stream( + transaction_stream_config: TransactionStreamConfig, + ) -> Result<(Streaming, String)> { info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self - .transaction_stream_config + stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - start_version = self.transaction_stream_config.starting_version, - end_version = self.transaction_stream_config.request_ending_version, + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, "[Parser] Connecting to GRPC stream", ); - let response = get_stream( - self.transaction_stream_config.clone(), - self.processor_name.to_string(), - ) - .await?; - let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { + let resp_stream = get_stream(transaction_stream_config.clone()).await?; + let connection_id = match resp_stream.metadata().get(GRPC_CONNECTION_ID) { Some(connection_id) => connection_id.to_str().unwrap().to_string(), None => "".to_string(), }; - self.connection_id = Some(connection_id); - self.resp_stream = Some(response.into_inner()); info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self - .transaction_stream_config + stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - connection_id = self.connection_id, - start_version = self.transaction_stream_config.starting_version, - end_version = self.transaction_stream_config.request_ending_version, + connection_id = connection_id, + start_version = transaction_stream_config.starting_version, + end_version = transaction_stream_config.request_ending_version, "[Parser] Successfully connected to GRPC stream", ); - Ok(()) + Ok((resp_stream.into_inner(), connection_id)) } /// Gets a batch of transactions from the stream. Batch size is set in the grpc server. @@ -421,17 +369,14 @@ impl TransactionStream { let txn_pb_res = match tokio::time::timeout( self.transaction_stream_config .indexer_grpc_response_item_timeout(), - self.resp_stream - .as_mut() - .expect("[Parser] GRPC stream is not initialized. Did you call init_stream?") - .next(), + self.stream.next(), ) .await { // Received datastream response Ok(response) => { match response { - Some(Ok(mut r)) => { + Some(Ok(r)) => { self.reconnection_retries = 0; let start_version = r.transactions.as_slice().first().unwrap().version; let start_txn_timestamp = @@ -616,17 +561,13 @@ impl TransactionStream { reconnection_retries = self.reconnection_retries, "[Parser] Reconnecting to GRPC stream" ); - let response = get_stream( - self.transaction_stream_config.clone(), - self.processor_name.to_string(), - ) - .await?; + let response = get_stream(self.transaction_stream_config.clone()).await?; let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { Some(connection_id) => connection_id.to_str().unwrap().to_string(), None => "".to_string(), }; - self.connection_id = Some(connection_id); - self.resp_stream = Some(response.into_inner()); + self.connection_id = connection_id; + self.stream = response.into_inner(); info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, @@ -644,6 +585,6 @@ impl TransactionStream { } pub async fn get_chain_id(self) -> Result { - get_chain_id(self.transaction_stream_config, self.processor_name).await + get_chain_id(self.transaction_stream_config).await } } diff --git a/rust/processor/src/utils/counters.rs b/rust/processor/src/utils/counters.rs index 023b44554..165d5f2f4 100644 --- a/rust/processor/src/utils/counters.rs +++ b/rust/processor/src/utils/counters.rs @@ -209,11 +209,9 @@ pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { /// Data gap warnings pub static PROCESSOR_DATA_GAP_COUNT: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "indexer_processor_data_gap_count", - "Data gap count", - &["processor_name"] - ) + register_int_gauge_vec!("indexer_processor_data_gap_count", "Data gap count", &[ + "processor_name" + ]) .unwrap() }); diff --git a/rust/processor/src/worker.rs b/rust/processor/src/worker.rs index 9ce354ce4..b4ef537d7 100644 --- a/rust/processor/src/worker.rs +++ b/rust/processor/src/worker.rs @@ -182,6 +182,7 @@ impl Worker { starting_version, request_ending_version: self.ending_version, auth_token: self.auth_token.clone(), + request_name_header: processor_name.to_string(), indexer_grpc_http2_ping_interval_secs: self .grpc_http2_config .indexer_grpc_http2_ping_interval_in_secs, @@ -197,7 +198,6 @@ impl Worker { // get the chain id let chain_id = aptos_indexer_transaction_stream::transaction_stream::get_chain_id( transaction_stream_config.clone(), - processor_name.to_string(), ) .await .expect("[Parser] Error getting chain id"); @@ -690,7 +690,7 @@ pub async fn fetch_transactions_from_transaction_stream( } } else { // We are breaking down a big batch into small batches; this involves an iterator - let average_size_in_bytes = size_in_bytes / num_filtered_txns as u64; + let average_size_in_bytes = size_in_bytes / num_filtered_txns; let txn_pb_chunked: Vec> = txn_pb .transactions @@ -729,7 +729,7 @@ pub async fn fetch_transactions_from_transaction_stream( // Log channel send metrics let duration_in_secs = txn_channel_send_latency.elapsed().as_secs_f64(); - send_ma.tick_now(num_txns as u64); + send_ma.tick_now(num_txns); let tps = send_ma.avg().ceil() as u64; let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; let channel_size = txn_sender.len(); @@ -752,7 +752,7 @@ pub async fn fetch_transactions_from_transaction_stream( .set(channel_size as i64); NUM_TRANSACTIONS_FILTERED_OUT_COUNT .with_label_values(&[&processor_name]) - .inc_by(num_filtered_txns as u64); + .inc_by(num_filtered_txns); }, Err(e) => { if transaction_stream.is_end_of_stream() { @@ -783,6 +783,13 @@ pub async fn fetch_transactions_from_transaction_stream( ); break; } else { + error!( + processor_name = processor_name, + stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), + error = ?e, + "[Parser] Error fetching transactions." + ); + // If there was an error fetching transactions, try to reconnect match transaction_stream.reconnect_to_grpc().await { Ok(_) => { From 682703fd67b479b9972ef0186602a98094ca244d Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Fri, 14 Jun 2024 19:08:08 -0400 Subject: [PATCH 11/13] remove rest of panics --- .../src/transaction_stream.rs | 71 +++++++++++++------ rust/processor/src/worker.rs | 12 ++-- 2 files changed, 56 insertions(+), 27 deletions(-) diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 5886603ce..232ccd532 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -48,6 +48,7 @@ pub struct TransactionsPBResponse { pub size_in_bytes: u64, } +/// Helper function to build a GRPC request for fetching transactions. pub fn grpc_request_builder( starting_version: u64, transactions_count: Option, @@ -72,6 +73,8 @@ pub fn grpc_request_builder( request } +/// Given a `TransactionStreamConfig`, this function will return a stream of transactions. +/// It also handles timeouts and retries. pub async fn get_stream( transaction_stream_config: TransactionStreamConfig, ) -> Result>> { @@ -234,6 +237,7 @@ pub async fn get_stream( } } +/// Helper function to get the chain id from the stream. pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> Result { info!( stream_address = transaction_stream_config @@ -294,6 +298,13 @@ pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> } } +/// TransactionStream is a struct that holds the state of the stream and provides methods to fetch transactions +/// from the stream. +/// - init_stream: Initializes the stream and returns the stream and connection id +/// - get_next_transaction_batch: Fetches the next batch of transactions from the stream +/// - is_end_of_stream: Checks if we've reached the end of the stream. This is determined by the ending version set in `TransactionStreamConfig` +/// - reconnect_to_grpc: Reconnects to the GRPC stream +/// - get_chain_id: Fetches the chain id from the stream pub struct TransactionStream { transaction_stream_config: TransactionStreamConfig, processor_name: String, @@ -361,9 +372,7 @@ impl TransactionStream { /// Returns /// - true if should continue fetching /// - false if we reached the end of the stream or there is an error and the loop should stop - pub async fn get_next_transaction_batch_with_reconnect( - &mut self, - ) -> Result { + pub async fn get_next_transaction_batch(&mut self) -> Result { let grpc_channel_recv_latency = std::time::Instant::now(); let txn_pb_res = match tokio::time::timeout( @@ -438,7 +447,7 @@ impl TransactionStream { current_fetched_version = start_version, "[Parser] Received batch with gap from GRPC stream" ); - panic!("[Parser] Received batch with gap from GRPC stream"); + return Err(anyhow!("Received batch with gap from GRPC stream")); } self.last_fetched_version = end_version as i64; @@ -531,24 +540,46 @@ impl TransactionStream { } } - pub async fn reconnect_to_grpc(&mut self) -> Result<()> { - // Sleep for 100ms between reconnect tries - // TODO: Turn this into exponential backoff - tokio::time::sleep(Duration::from_millis(100)).await; + pub async fn reconnect_to_grpc_with_retries(&mut self) -> Result<()> { + let mut reconnection_retries = 0; - if self.reconnection_retries >= RECONNECTION_MAX_RETRIES { - error!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, - stream_address = self - .transaction_stream_config - .indexer_grpc_data_service_address - .to_string(), - "[Parser] Reconnected more than 100 times. Will not retry.", - ); - panic!("[Parser] Reconnected more than 100 times. Will not retry.") + loop { + // Sleep for 100ms between reconnect tries + // TODO: Turn this into exponential backoff + tokio::time::sleep(Duration::from_millis(100)).await; + + reconnection_retries += 1; + + if reconnection_retries >= RECONNECTION_MAX_RETRIES { + error!( + stream_address = self + .transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + "[Parser] Reconnected more than 100 times. Will not retry.", + ); + break Err(anyhow!("Reconnected more than 100 times. Will not retry.")); + } + + match self.reconnect_to_grpc().await { + Ok(_) => { + break Ok(()); + }, + Err(e) => { + error!( + stream_address = self.transaction_stream_config + .indexer_grpc_data_service_address + .to_string(), + error = ?e, + "[Parser] Error reconnecting to GRPC stream. Retrying..." + ); + continue; + }, + } } - self.reconnection_retries += 1; + } + + pub async fn reconnect_to_grpc(&mut self) -> Result<()> { info!( processor_name = self.processor_name, service_type = PROCESSOR_SERVICE_TYPE, diff --git a/rust/processor/src/worker.rs b/rust/processor/src/worker.rs index b4ef537d7..8f8473733 100644 --- a/rust/processor/src/worker.rs +++ b/rust/processor/src/worker.rs @@ -654,9 +654,7 @@ pub async fn fetch_transactions_from_transaction_stream( // 2. If we specified an end version and we hit that, we will stop fetching, but we will make sure that // all existing transactions are processed. loop { - let transactions_res = transaction_stream - .get_next_transaction_batch_with_reconnect() - .await; + let transactions_res = transaction_stream.get_next_transaction_batch().await; match transactions_res { Ok(mut txn_pb) => { @@ -785,13 +783,14 @@ pub async fn fetch_transactions_from_transaction_stream( } else { error!( processor_name = processor_name, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), error = ?e, "[Parser] Error fetching transactions." ); // If there was an error fetching transactions, try to reconnect - match transaction_stream.reconnect_to_grpc().await { + match transaction_stream.reconnect_to_grpc_with_retries().await { Ok(_) => { info!( processor_name = processor_name, @@ -799,7 +798,7 @@ pub async fn fetch_transactions_from_transaction_stream( stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - "[Parser] Successfully reconnected to GRPC stream." + "[Parser] Successfully reconnected transaction stream." ); }, Err(e) => { @@ -810,9 +809,8 @@ pub async fn fetch_transactions_from_transaction_stream( .indexer_grpc_data_service_address .to_string(), error = ?e, - "[Parser] Error reconnecting to GRPC stream." + "[Parser] Error reconnecting transaction stream." ); - panic!("[Parser] Error reconnecting to GRPC stream.") }, } } From 2699eeeb7fb31f913380af5028f4b7216221a665 Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Sun, 16 Jun 2024 17:31:20 -0400 Subject: [PATCH 12/13] Remove metrics --- .../src/transaction_stream.rs | 65 +------------ .../src/utils/counters.rs | 95 ------------------- .../src/utils/mod.rs | 1 - rust/processor/src/worker.rs | 27 +++++- 4 files changed, 29 insertions(+), 159 deletions(-) delete mode 100644 rust/aptos-indexer-transaction-stream/src/utils/counters.rs diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 232ccd532..743d5acca 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -1,13 +1,4 @@ -use crate::{ - config::TransactionStreamConfig, - utils::{ - counters::{ - ProcessorStep, LATEST_PROCESSED_VERSION, NUM_TRANSACTIONS_PROCESSED_COUNT, - PROCESSED_BYTES_COUNT, TRANSACTION_UNIX_TIMESTAMP, - }, - util::{timestamp_to_iso, timestamp_to_unixtime}, - }, -}; +use crate::{config::TransactionStreamConfig, utils::util::timestamp_to_iso}; use anyhow::{anyhow, Result}; use aptos_moving_average::MovingAverage; use aptos_protos::{ @@ -34,8 +25,8 @@ pub const RECONNECTION_MAX_RETRIES: u64 = 5; /// 256MB pub const MAX_RESPONSE_SIZE: usize = 1024 * 1024 * 256; -const PROCESSOR_SERVICE_TYPE: &str = "processor"; - +/// TransactionsPBResponse is a struct that holds the transactions fetched from the stream. +/// It also includes some contextual information about the transactions. #[derive(Clone)] pub struct TransactionsPBResponse { pub transactions: Vec, @@ -307,7 +298,6 @@ pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> /// - get_chain_id: Fetches the chain id from the stream pub struct TransactionStream { transaction_stream_config: TransactionStreamConfig, - processor_name: String, stream: Streaming, connection_id: String, next_version_to_fetch: u64, @@ -317,14 +307,10 @@ pub struct TransactionStream { } impl TransactionStream { - pub async fn new( - transaction_stream_config: TransactionStreamConfig, - processor_name: String, - ) -> Result { + pub async fn new(transaction_stream_config: TransactionStreamConfig) -> Result { let (stream, connection_id) = Self::init_stream(transaction_stream_config.clone()).await?; Ok(Self { transaction_stream_config: transaction_stream_config.clone(), - processor_name, stream, connection_id, next_version_to_fetch: transaction_stream_config.starting_version, @@ -402,18 +388,7 @@ impl TransactionStream { let duration_in_secs = grpc_channel_recv_latency.elapsed().as_secs_f64(); self.fetch_ma.tick_now(num_txns as u64); - // Filter out the txns we don't care about - // r.transactions - // .retain(|txn| self.transaction_filter.include(txn)); - - // let num_txn_post_filter = r.transactions.len(); - // let num_filtered_txns = num_txns - num_txn_post_filter; - let step = ProcessorStep::ReceivedTxnsFromGrpc.get_step(); - let label = ProcessorStep::ReceivedTxnsFromGrpc.get_label(); - info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = self .transaction_stream_config .indexer_grpc_data_service_address @@ -435,9 +410,7 @@ impl TransactionStream { duration_in_secs, tps = self.fetch_ma.avg().ceil() as u64, bytes_per_sec = size_in_bytes as f64 / duration_in_secs, - step, - "{}", - label, + "Received transactions from GRPC.", ); if self.last_fetched_version + 1 != start_version as i64 { @@ -451,24 +424,6 @@ impl TransactionStream { } self.last_fetched_version = end_version as i64; - LATEST_PROCESSED_VERSION - .with_label_values(&[&self.processor_name, step, label, "-"]) - .set(end_version as i64); - TRANSACTION_UNIX_TIMESTAMP - .with_label_values(&[&self.processor_name, step, label, "-"]) - .set( - start_txn_timestamp - .as_ref() - .map(timestamp_to_unixtime) - .unwrap_or_default(), - ); - PROCESSED_BYTES_COUNT - .with_label_values(&[&self.processor_name, step, label, "-"]) - .inc_by(size_in_bytes); - NUM_TRANSACTIONS_PROCESSED_COUNT - .with_label_values(&[&self.processor_name, step, label, "-"]) - .inc_by(end_version - start_version + 1); - let txn_pb = TransactionsPBResponse { transactions: r.transactions, chain_id, @@ -484,8 +439,6 @@ impl TransactionStream { // Error receiving datastream response Some(Err(rpc_error)) => { tracing::warn!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.transaction_stream_config.indexer_grpc_data_service_address.to_string(), self.connection_id, start_version = self.transaction_stream_config.starting_version, @@ -498,8 +451,6 @@ impl TransactionStream { // Stream is finished None => { tracing::warn!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = self .transaction_stream_config .indexer_grpc_data_service_address @@ -516,8 +467,6 @@ impl TransactionStream { // Timeout receiving datastream response Err(e) => { tracing::warn!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = self.transaction_stream_config.indexer_grpc_data_service_address.to_string(), connection_id = self.connection_id, start_version = self.transaction_stream_config.starting_version, @@ -581,8 +530,6 @@ impl TransactionStream { pub async fn reconnect_to_grpc(&mut self) -> Result<()> { info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = self .transaction_stream_config .indexer_grpc_data_service_address @@ -600,8 +547,6 @@ impl TransactionStream { self.connection_id = connection_id; self.stream = response.into_inner(); info!( - processor_name = self.processor_name, - service_type = PROCESSOR_SERVICE_TYPE, stream_address = self .transaction_stream_config .indexer_grpc_data_service_address diff --git a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs b/rust/aptos-indexer-transaction-stream/src/utils/counters.rs deleted file mode 100644 index 959cd196b..000000000 --- a/rust/aptos-indexer-transaction-stream/src/utils/counters.rs +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -use once_cell::sync::Lazy; -use prometheus::{ - register_gauge_vec, register_int_counter_vec, register_int_gauge_vec, GaugeVec, IntCounterVec, - IntGaugeVec, -}; - -pub enum ProcessorStep { - ReceivedTxnsFromGrpc, - // Received transactions from GRPC. Sending transactions to channel. - ProcessedBatch, - // Processor finished processing one batch of transaction - ProcessedMultipleBatches, // Processor finished processing multiple batches of transactions -} - -impl ProcessorStep { - pub fn get_step(&self) -> &'static str { - match self { - ProcessorStep::ReceivedTxnsFromGrpc => "1", - ProcessorStep::ProcessedBatch => "2", - ProcessorStep::ProcessedMultipleBatches => "3", - } - } - - pub fn get_label(&self) -> &'static str { - match self { - ProcessorStep::ReceivedTxnsFromGrpc => { - "[Parser] Received transactions from GRPC. Sending transactions to channel." - }, - ProcessorStep::ProcessedBatch => { - "[Parser] Processor finished processing one batch of transaction" - }, - ProcessorStep::ProcessedMultipleBatches => { - "[Parser] Processor finished processing multiple batches of transactions" - }, - } - } -} - -pub const TRANSACTION_STREAM_METRICS_PREFIX: &str = "transaction_stream_"; - -/// These metrics are temporary (suffixed with _temp to avoid conflict with metrics in processor crate) -/// They're only defined in this crate for backwards compatibility before we migrate over to -/// using the instrumentation provided by SDK - -/// Max version processed -pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - format!("{}_latest_version", TRANSACTION_STREAM_METRICS_PREFIX), - "Latest version a processor has fully consumed", - &["processor_name", "step", "message", "task_index"] - ) - .unwrap() -}); - -/// Count of bytes processed. -pub static PROCESSED_BYTES_COUNT: Lazy = Lazy::new(|| { - register_int_counter_vec!( - format!( - "{}_processed_bytes_count", - TRANSACTION_STREAM_METRICS_PREFIX - ), - "Count of bytes processed", - &["processor_name", "step", "message", "task_index"] - ) - .unwrap() -}); - -/// Count of transactions processed. -pub static NUM_TRANSACTIONS_PROCESSED_COUNT: Lazy = Lazy::new(|| { - register_int_counter_vec!( - format!( - "{}_num_transactions_processed_count", - TRANSACTION_STREAM_METRICS_PREFIX - ), - "Number of transactions processed", - &["processor_name", "step", "message", "task_index"] - ) - .unwrap() -}); - -/// Transaction timestamp in unixtime -pub static TRANSACTION_UNIX_TIMESTAMP: Lazy = Lazy::new(|| { - register_gauge_vec!( - format!( - "{}_transaction_unix_timestamp", - TRANSACTION_STREAM_METRICS_PREFIX - ), - "Transaction timestamp in unixtime", - &["processor_name", "step", "message", "task_index"] - ) - .unwrap() -}); diff --git a/rust/aptos-indexer-transaction-stream/src/utils/mod.rs b/rust/aptos-indexer-transaction-stream/src/utils/mod.rs index 78b4ba670..812d1edf2 100644 --- a/rust/aptos-indexer-transaction-stream/src/utils/mod.rs +++ b/rust/aptos-indexer-transaction-stream/src/utils/mod.rs @@ -1,2 +1 @@ -pub mod counters; pub mod util; diff --git a/rust/processor/src/worker.rs b/rust/processor/src/worker.rs index 8f8473733..cd94084b3 100644 --- a/rust/processor/src/worker.rs +++ b/rust/processor/src/worker.rs @@ -630,8 +630,7 @@ pub async fn fetch_transactions_from_transaction_stream( pb_channel_txn_chunk_size: usize, ) { // Initialize the stream - let transaction_stream_res = - TransactionStream::new(transaction_stream_config.clone(), processor_name.clone()).await; + let transaction_stream_res = TransactionStream::new(transaction_stream_config.clone()).await; let mut transaction_stream = match transaction_stream_res { Ok(stream) => stream, @@ -662,6 +661,7 @@ pub async fn fetch_transactions_from_transaction_stream( let end_version = txn_pb.transactions.last().unwrap().version; let size_in_bytes = txn_pb.size_in_bytes; let num_txns = end_version - start_version + 1; + let start_txn_timestamp = txn_pb.start_txn_timestamp.clone(); // Filter out the txns we don't care about txn_pb @@ -731,8 +731,12 @@ pub async fn fetch_transactions_from_transaction_stream( let tps = send_ma.avg().ceil() as u64; let bytes_per_sec = size_in_bytes as f64 / duration_in_secs; let channel_size = txn_sender.len(); + + let step = ProcessorStep::ReceivedTxnsFromGrpc.get_step(); + let label = ProcessorStep::ReceivedTxnsFromGrpc.get_label(); debug!( - processor_name.service_type = PROCESSOR_SERVICE_TYPE, + processor_name, + service_type = PROCESSOR_SERVICE_TYPE, stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), @@ -745,6 +749,23 @@ pub async fn fetch_transactions_from_transaction_stream( tps, "[Parser] Successfully sent transactions to channel." ); + LATEST_PROCESSED_VERSION + .with_label_values(&[&processor_name, step, label, "-"]) + .set(end_version as i64); + TRANSACTION_UNIX_TIMESTAMP + .with_label_values(&[&processor_name, step, label, "-"]) + .set( + start_txn_timestamp + .as_ref() + .map(timestamp_to_unixtime) + .unwrap_or_default(), + ); + PROCESSED_BYTES_COUNT + .with_label_values(&[&processor_name, step, label, "-"]) + .inc_by(size_in_bytes); + NUM_TRANSACTIONS_PROCESSED_COUNT + .with_label_values(&[&processor_name, step, label, "-"]) + .inc_by(end_version - start_version + 1); FETCHER_THREAD_CHANNEL_SIZE .with_label_values(&[&processor_name]) .set(channel_size as i64); From 3c8896b489062ce0510c5e4162366db753b59499 Mon Sep 17 00:00:00 2001 From: rtso <8248583+rtso@users.noreply.github.com> Date: Sun, 16 Jun 2024 17:32:46 -0400 Subject: [PATCH 13/13] Rename logs --- .../src/transaction_stream.rs | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs index 743d5acca..e0017fddb 100644 --- a/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs +++ b/rust/aptos-indexer-transaction-stream/src/transaction_stream.rs @@ -75,7 +75,7 @@ pub async fn get_stream( .to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, - "[Parser] Setting up rpc channel" + "[Transaction Stream] Setting up rpc channel" ); let channel = tonic::transport::Channel::from_shared( @@ -84,7 +84,7 @@ pub async fn get_stream( .to_string(), ) .expect( - "[Parser] Failed to build GRPC channel, perhaps because the data service URL is invalid", + "[Transaction Stream] Failed to build GRPC channel, perhaps because the data service URL is invalid", ) .http2_keep_alive_interval(transaction_stream_config.indexer_grpc_http2_ping_interval()) .keep_alive_timeout(transaction_stream_config.indexer_grpc_http2_ping_timeout()); @@ -98,7 +98,7 @@ pub async fn get_stream( let config = tonic::transport::channel::ClientTlsConfig::new(); channel .tls_config(config) - .expect("[Parser] Failed to create TLS config") + .expect("[Transaction Stream] Failed to create TLS config") } else { channel }; @@ -109,7 +109,7 @@ pub async fn get_stream( .to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, - "[Parser] Setting up GRPC client" + "[Transaction Stream] Setting up GRPC client" ); // TODO: move this to a config file @@ -130,7 +130,7 @@ pub async fn get_stream( start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, error = ?e, - "[Parser] Error connecting to GRPC client" + "[Transaction Stream] Error connecting to GRPC client" ); connect_retries += 1; if connect_retries >= RECONNECTION_MAX_RETRIES { @@ -145,7 +145,7 @@ pub async fn get_stream( end_version = transaction_stream_config.request_ending_version, retries = connect_retries, error = ?e, - "[Parser] Timed out connecting to GRPC client" + "[Transaction Stream] Timed out connecting to GRPC client" ); connect_retries += 1; if connect_retries >= RECONNECTION_MAX_RETRIES { @@ -173,7 +173,7 @@ pub async fn get_stream( start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, num_of_transactions = ?count, - "[Parser] Setting up GRPC stream", + "[Transaction Stream] Setting up GRPC stream", ); // TODO: move this to a config file @@ -202,7 +202,7 @@ pub async fn get_stream( start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, error = ?e, - "[Parser] Error making grpc request. Retrying..." + "[Transaction Stream] Error making grpc request. Retrying..." ); connect_retries += 1; if connect_retries >= RECONNECTION_MAX_RETRIES { @@ -217,7 +217,7 @@ pub async fn get_stream( end_version = transaction_stream_config.request_ending_version, retries = connect_retries, error = ?e, - "[Parser] Timeout making grpc request. Retrying...", + "[Transaction Stream] Timeout making grpc request. Retrying...", ); connect_retries += 1; if connect_retries >= RECONNECTION_MAX_RETRIES { @@ -234,7 +234,7 @@ pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - "[Parser] Connecting to GRPC stream to get chain id", + "[Transaction Stream] Connecting to GRPC stream to get chain id", ); let transaction_stream_config_for_chain_id = TransactionStreamConfig { @@ -252,7 +252,7 @@ pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - connection_id, "[Parser] Successfully connected to GRPC stream to get chain id", + connection_id, "[Transaction Stream] Successfully connected to GRPC stream to get chain id", ); match resp_stream.next().await { @@ -263,7 +263,7 @@ pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - connection_id, "[Parser] Chain Id doesn't exist." + connection_id, "[Transaction Stream] Chain Id doesn't exist." ); Err(anyhow!("Chain Id doesn't exist")) }, @@ -273,7 +273,7 @@ pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> stream_address = transaction_stream_config.indexer_grpc_data_service_address.to_string(), connection_id, error = ?rpc_error, - "[Parser] Error receiving datastream response for chain id" + "[Transaction Stream] Error receiving datastream response for chain id" ); Err(anyhow!("Error receiving datastream response for chain id").context(rpc_error)) }, @@ -282,7 +282,8 @@ pub async fn get_chain_id(transaction_stream_config: TransactionStreamConfig) -> stream_address = transaction_stream_config .indexer_grpc_data_service_address .to_string(), - connection_id, "[Parser] Stream ended before getting response fo for chain id" + connection_id, + "[Transaction Stream] Stream ended before getting response fo for chain id" ); Err(anyhow!("Stream ended before getting response for chain id")) }, @@ -329,7 +330,7 @@ impl TransactionStream { .to_string(), start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, - "[Parser] Connecting to GRPC stream", + "[Transaction Stream] Connecting to GRPC stream", ); let resp_stream = get_stream(transaction_stream_config.clone()).await?; let connection_id = match resp_stream.metadata().get(GRPC_CONNECTION_ID) { @@ -343,7 +344,7 @@ impl TransactionStream { connection_id = connection_id, start_version = transaction_stream_config.starting_version, end_version = transaction_stream_config.request_ending_version, - "[Parser] Successfully connected to GRPC stream", + "[Transaction Stream] Successfully connected to GRPC stream", ); Ok((resp_stream.into_inner(), connection_id)) } @@ -383,7 +384,9 @@ impl TransactionStream { self.next_version_to_fetch = end_version + 1; let size_in_bytes = r.encoded_len() as u64; - let chain_id: u64 = r.chain_id.expect("[Parser] Chain Id doesn't exist."); + let chain_id: u64 = r + .chain_id + .expect("[Transaction Stream] Chain Id doesn't exist."); let num_txns = r.transactions.len(); let duration_in_secs = grpc_channel_recv_latency.elapsed().as_secs_f64(); self.fetch_ma.tick_now(num_txns as u64); @@ -410,7 +413,7 @@ impl TransactionStream { duration_in_secs, tps = self.fetch_ma.avg().ceil() as u64, bytes_per_sec = size_in_bytes as f64 / duration_in_secs, - "Received transactions from GRPC.", + "[Transaction Stream] Received transactions from GRPC.", ); if self.last_fetched_version + 1 != start_version as i64 { @@ -418,7 +421,7 @@ impl TransactionStream { batch_start_version = self.last_fetched_version + 1, self.last_fetched_version, current_fetched_version = start_version, - "[Parser] Received batch with gap from GRPC stream" + "[Transaction Stream] Received batch with gap from GRPC stream" ); return Err(anyhow!("Received batch with gap from GRPC stream")); } @@ -444,7 +447,7 @@ impl TransactionStream { start_version = self.transaction_stream_config.starting_version, end_version = self.transaction_stream_config.request_ending_version, error = ?rpc_error, - "[Parser] Error receiving datastream response." + "[Transaction Stream] Error receiving datastream response." ); Err(anyhow!("Error receiving datastream response")) }, @@ -458,7 +461,7 @@ impl TransactionStream { connection_id = self.connection_id, start_version = self.transaction_stream_config.starting_version, end_version = self.transaction_stream_config.request_ending_version, - "[Parser] Stream ended." + "[Transaction Stream] Stream ended." ); Err(anyhow!("Stream ended")) }, @@ -472,7 +475,7 @@ impl TransactionStream { start_version = self.transaction_stream_config.starting_version, end_version = self.transaction_stream_config.request_ending_version, error = ?e, - "[Parser] Timeout receiving datastream response." + "[Transaction Stream] Timeout receiving datastream response." ); Err(anyhow!("Timeout receiving datastream response")) }, @@ -505,7 +508,7 @@ impl TransactionStream { .transaction_stream_config .indexer_grpc_data_service_address .to_string(), - "[Parser] Reconnected more than 100 times. Will not retry.", + "[Transaction Stream] Reconnected more than 100 times. Will not retry.", ); break Err(anyhow!("Reconnected more than 100 times. Will not retry.")); } @@ -520,7 +523,7 @@ impl TransactionStream { .indexer_grpc_data_service_address .to_string(), error = ?e, - "[Parser] Error reconnecting to GRPC stream. Retrying..." + "[Transaction Stream] Error reconnecting to GRPC stream. Retrying..." ); continue; }, @@ -537,7 +540,7 @@ impl TransactionStream { starting_version = self.next_version_to_fetch, ending_version = self.transaction_stream_config.request_ending_version, reconnection_retries = self.reconnection_retries, - "[Parser] Reconnecting to GRPC stream" + "[Transaction Stream] Reconnecting to GRPC stream" ); let response = get_stream(self.transaction_stream_config.clone()).await?; let connection_id = match response.metadata().get(GRPC_CONNECTION_ID) { @@ -555,7 +558,7 @@ impl TransactionStream { starting_version = self.next_version_to_fetch, ending_version = self.transaction_stream_config.request_ending_version, reconnection_retries = self.reconnection_retries, - "[Parser] Successfully reconnected to GRPC stream" + "[Transaction Stream] Successfully reconnected to GRPC stream" ); Ok(()) }