From 4b32672b44bef2e71cf6f82e4ab780bf44e8fcde Mon Sep 17 00:00:00 2001 From: Daniel Porteous Date: Fri, 14 Jun 2024 13:47:31 +0200 Subject: [PATCH] [Data Service] Implement simple upstream transaction filtering There are two types of transaction filtering we will support in the future: 1. Per stream configuration: The downstream declares what txns they want to receive. 2. Global configuration: At the data service level we refuse to include full txns for all streams. This PR implements the second of these, using @CapCap's work here: https://github.com/aptos-labs/aptos-indexer-processors/pull/398. Rather than not sending txns at all if they match the blocklist filters, we just omit the writesets and events. Not sending the txns entirely would cause issues with processors, which today assume that they will receive all txns. --- Cargo.lock | 138 ++++++++++++------ Cargo.toml | 3 +- config/src/config/persistable_config.rs | 18 ++- .../indexer-grpc-data-service/Cargo.toml | 1 + 4 files changed, 116 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44d7536b9fe377..479f09c6d154eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,7 +315,7 @@ dependencies = [ "self_update", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "server-framework", "shadow-rs", "tempfile", @@ -574,7 +574,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "thiserror", "tokio", "tokio-io-timeout", @@ -825,7 +825,7 @@ dependencies = [ "serde", "serde_json", "serde_merge", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tempfile", "thiserror", "url", @@ -902,7 +902,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "sha3 0.9.1", "strum_macros 0.24.3", "tempfile", @@ -1590,7 +1590,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tokio", "url", ] @@ -1680,7 +1680,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tempfile", "termcolor", "thiserror", @@ -1710,7 +1710,7 @@ dependencies = [ "rand 0.7.3", "random_word", "reqwest", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tokio", "url", ] @@ -1898,7 +1898,7 @@ dependencies = [ "bcs 0.1.4", "rand 0.7.3", "serde", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", ] [[package]] @@ -2005,6 +2005,7 @@ dependencies = [ "tonic 0.11.0", "tonic-reflection", "tracing", + "transaction-filter", "uuid", ] @@ -2136,7 +2137,7 @@ dependencies = [ "clap 4.4.14", "prometheus", "serde", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tempfile", "tokio", "toml 0.7.8", @@ -2893,7 +2894,7 @@ dependencies = [ "futures", "once_cell", "rand 0.7.3", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tokio", "url", ] @@ -2996,7 +2997,7 @@ dependencies = [ "rstack-self", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tokio", "url", ] @@ -3023,7 +3024,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "thiserror", "tokio", "url", @@ -3289,7 +3290,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "sha3 0.9.1", "strum 0.24.1", "strum_macros 0.24.3", @@ -3394,7 +3395,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tokio", "url", "warp", @@ -3531,7 +3532,7 @@ dependencies = [ "once_cell", "serde-generate", "serde-reflection", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tempfile", "textwrap 0.15.2", "which", @@ -3881,7 +3882,7 @@ dependencies = [ "reqwest-retry", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "thiserror", "tokio", "tracing", @@ -4122,7 +4123,7 @@ dependencies = [ "serde_bytes", "serde_json", "serde_with", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "strum 0.24.1", "strum_macros 0.24.3", "thiserror", @@ -6655,12 +6656,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.3" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ - "darling_core 0.20.3", - "darling_macro 0.20.3", + "darling_core 0.20.9", + "darling_macro 0.20.9", ] [[package]] @@ -6693,15 +6694,15 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim 0.11.1", "syn 2.0.48", ] @@ -6729,11 +6730,11 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ - "darling_core 0.20.3", + "darling_core 0.20.9", "quote", "syn 2.0.48", ] @@ -6878,6 +6879,37 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "derive_builder" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" +dependencies = [ + "darling 0.20.9", + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" +dependencies = [ + "derive_builder_core", + "syn 2.0.48", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -8196,7 +8228,7 @@ dependencies = [ "rand 0.7.3", "serde", "serde-reflection", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", ] [[package]] @@ -10105,9 +10137,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -10775,7 +10807,7 @@ dependencies = [ "petgraph 0.5.1", "regex", "serde", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "sha2 0.9.9", "tempfile", "termcolor", @@ -12403,7 +12435,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "serde_yaml 0.9.30", + "serde_yaml 0.9.34+deprecated", "thiserror", "tokio", "url", @@ -14314,7 +14346,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" dependencies = [ - "darling 0.20.3", + "darling 0.20.9", "proc-macro2", "quote", "syn 2.0.48", @@ -14334,9 +14366,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.30" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ "indexmap 2.2.5", "itoa", @@ -14762,7 +14794,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.8.26", + "serde_yaml 0.9.34+deprecated", "tokio", "url", "walkdir", @@ -14921,6 +14953,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "structopt" version = "0.3.26" @@ -15307,18 +15345,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", @@ -15947,6 +15985,22 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "transaction-filter" +version = "0.1.0" +source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=13a63d8e3c07cbda36907ca2f1ee1a323e910897#13a63d8e3c07cbda36907ca2f1ee1a323e910897" +dependencies = [ + "anyhow", + "aptos-protos 1.3.0 (git+https://github.com/aptos-labs/aptos-core.git?tag=aptos-node-v1.12.1)", + "derive_builder", + "memchr", + "prost 0.12.3", + "serde", + "serde_json", + "serde_yaml 0.9.34+deprecated", + "thiserror", +] + [[package]] name = "treediff" version = "3.0.2" @@ -16248,9 +16302,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" diff --git a/Cargo.toml b/Cargo.toml index 76a3d4399b09ed..f6d2fda8407d02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -473,6 +473,7 @@ ark-groth16 = "0.4.0" ark-serialize = "0.4.0" ark-std = { version = "0.4.0", features = ["getrandom"] } aptos-moving-average = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "4801acae7aea30d7e96bbfbe5ec5b04056dfa4cf" } +transaction-filter = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "13a63d8e3c07cbda36907ca2f1ee1a323e910897" } assert_approx_eq = "1.1.0" assert_unordered = "0.3.5" async-channel = "1.7.1" @@ -724,7 +725,7 @@ serde-name = "0.1.1" serde-generate = { git = "https://github.com/aptos-labs/serde-reflection", rev = "73b6bbf748334b71ff6d7d09d06a29e3062ca075" } serde-reflection = { git = "https://github.com/aptos-labs/serde-reflection", rev = "73b6bbf748334b71ff6d7d09d06a29e3062ca075" } serde_with = "3.4.0" -serde_yaml = "0.8.24" +serde_yaml = "0.9.34" shadow-rs = "0.16.2" simplelog = "0.9.0" smallbitvec = "2.5.1" diff --git a/config/src/config/persistable_config.rs b/config/src/config/persistable_config.rs index 728f372d15da5f..ca9d811a932a60 100644 --- a/config/src/config/persistable_config.rs +++ b/config/src/config/persistable_config.rs @@ -22,7 +22,7 @@ pub trait PersistableConfig: Serialize + DeserializeOwned { /// Save the config to disk at the given output path fn save_config>(&self, output_file: P) -> Result<(), Error> { // Serialize the config to a string - let serialized_config = serde_yaml::to_vec(&self) + let serialized_config = serde_yaml_to_vec(&self) .map_err(|e| Error::Yaml(output_file.as_ref().to_str().unwrap().to_string(), e))?; Self::write_file(serialized_config, output_file) @@ -58,3 +58,19 @@ pub trait PersistableConfig: Serialize + DeserializeOwned { // We only implement PersistableConfig for the configs that should be read/written to disk impl PersistableConfig for NodeConfig {} impl PersistableConfig for SafetyRulesConfig {} + +/// Serialize the given data structure as a YAML byte vector. +/// +/// Serialization can fail if `T`'s implementation of `Serialize` decides to +/// return an error. +/// +/// serde_yaml 0.9.x doesn't have this function so we reimplement it here as it existed +/// in serde_yaml 0.8.x. +pub fn serde_yaml_to_vec(value: &T) -> serde_yaml::Result> +where + T: serde::ser::Serialize, +{ + let mut vec = Vec::with_capacity(128); + serde_yaml::to_writer(&mut vec, value)?; + Ok(vec) +} diff --git a/ecosystem/indexer-grpc/indexer-grpc-data-service/Cargo.toml b/ecosystem/indexer-grpc/indexer-grpc-data-service/Cargo.toml index c378e10000297c..780909520fd49a 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-data-service/Cargo.toml +++ b/ecosystem/indexer-grpc/indexer-grpc-data-service/Cargo.toml @@ -31,6 +31,7 @@ tokio = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } tonic-reflection = { workspace = true } +transaction-filter = { workspace = true } tracing = { workspace = true } uuid = { workspace = true }