From b55ad61f7028b1875719b7c706b378e3b34b85cd Mon Sep 17 00:00:00 2001 From: Guoteng Rao <3603304+grao1991@users.noreply.github.com> Date: Sun, 29 Oct 2023 13:26:23 -0700 Subject: [PATCH] [Tooling] Combine some tools to a single binary. (#10709) --- .../workflows/workflow-run-module-verify.yaml | 2 +- .../workflows/workflow-run-replay-verify.yaml | 2 +- Cargo.lock | 85 +++-- Cargo.toml | 11 +- aptos-move/aptos-debugger/Cargo.toml | 23 +- aptos-move/aptos-debugger/README.md | 176 ---------- .../aptos-debugger/src/aptos_debugger.rs | 318 ++++++++++++++++++ .../aptos-debugger/src/bcs_txn_decoder.rs | 81 +++++ .../aptos-debugger/src/bin/bcs-txn-decoder.rs | 82 ----- aptos-move/aptos-debugger/src/lib.rs | 261 +------------- aptos-move/aptos-debugger/src/main.rs | 61 ---- crates/aptos-debugger/Cargo.toml | 22 ++ crates/aptos-debugger/src/lib.rs | 31 ++ .../aptos-debugger}/src/main.rs | 5 +- crates/aptos/Cargo.toml | 2 +- crates/aptos/src/common/types.rs | 2 +- docker/builder/README.md | 6 +- docker/builder/build-tools.sh | 6 +- docker/builder/tools.Dockerfile | 3 +- docker/builder/validator-testing.Dockerfile | 5 +- docker/builder/validator.Dockerfile | 5 +- .../compose/data-restore/docker-compose.yaml | 2 +- execution/db-bootstrapper/Cargo.toml | 26 -- .../src/bin/aptos-db-bootstrapper.rs | 118 ------- storage/README.md | 12 +- storage/aptosdb/Cargo.toml | 5 - storage/aptosdb/src/db_debugger/main.rs | 10 - storage/db-tool/Cargo.toml | 3 + storage/db-tool/src/bootstrap.rs | 118 +++++++ storage/db-tool/src/debugger.rs | 28 -- storage/db-tool/src/lib.rs | 22 +- .../fullnode/templates/backup-compaction.yaml | 5 +- .../fullnode/templates/backup-verify.yaml | 3 +- terraform/helm/fullnode/templates/backup.yaml | 3 +- .../helm/fullnode/templates/restore.yaml | 2 +- testsuite/module_verify.py | 3 +- testsuite/module_verify_run_local.py | 2 +- testsuite/replay_verify.py | 5 +- testsuite/replay_verify_run_local.py | 2 +- testsuite/smoke-test/Cargo.toml | 2 +- .../smoke-test/src/aptos/mint_transfer.rs | 2 +- testsuite/smoke-test/src/genesis.rs | 9 +- testsuite/smoke-test/src/storage.rs | 21 +- testsuite/verify_core/common.py | 3 +- 44 files changed, 705 insertions(+), 890 deletions(-) delete mode 100644 aptos-move/aptos-debugger/README.md create mode 100644 aptos-move/aptos-debugger/src/aptos_debugger.rs create mode 100644 aptos-move/aptos-debugger/src/bcs_txn_decoder.rs delete mode 100644 aptos-move/aptos-debugger/src/bin/bcs-txn-decoder.rs delete mode 100644 aptos-move/aptos-debugger/src/main.rs create mode 100644 crates/aptos-debugger/Cargo.toml create mode 100644 crates/aptos-debugger/src/lib.rs rename {storage/db-tool => crates/aptos-debugger}/src/main.rs (81%) delete mode 100644 execution/db-bootstrapper/Cargo.toml delete mode 100644 execution/db-bootstrapper/src/bin/aptos-db-bootstrapper.rs delete mode 100644 storage/aptosdb/src/db_debugger/main.rs create mode 100644 storage/db-tool/src/bootstrap.rs delete mode 100644 storage/db-tool/src/debugger.rs diff --git a/.github/workflows/workflow-run-module-verify.yaml b/.github/workflows/workflow-run-module-verify.yaml index c9e4c76ab1d04..2dd3a9ac7b0f4 100644 --- a/.github/workflows/workflow-run-module-verify.yaml +++ b/.github/workflows/workflow-run-module-verify.yaml @@ -60,7 +60,7 @@ jobs: - name: Build CLI binaries in release mode shell: bash - run: cargo build --release -p aptos-db-tool + run: cargo build --release -p aptos-debugger - name: Run module-verify in parallel shell: bash diff --git a/.github/workflows/workflow-run-replay-verify.yaml b/.github/workflows/workflow-run-replay-verify.yaml index 764d5984ddf0e..575d3b437f48f 100644 --- a/.github/workflows/workflow-run-replay-verify.yaml +++ b/.github/workflows/workflow-run-replay-verify.yaml @@ -103,7 +103,7 @@ jobs: - name: Build CLI binaries in release mode shell: bash - run: cargo build --release -p aptos-db-tool + run: cargo build --release -p aptos-debugger - name: Run replay-verify in parallel shell: bash diff --git a/Cargo.lock b/Cargo.lock index 70234e600a894..1e9c4620564a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,7 +226,6 @@ dependencies = [ "aptos-cli-common", "aptos-config", "aptos-crypto", - "aptos-debugger", "aptos-faucet-core", "aptos-framework", "aptos-gas-profiling", @@ -239,6 +238,7 @@ dependencies = [ "aptos-keygen", "aptos-ledger", "aptos-logger", + "aptos-move-debugger", "aptos-network-checker", "aptos-node", "aptos-protos 1.1.2", @@ -1069,23 +1069,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "aptos-db-bootstrapper" -version = "0.1.0" -dependencies = [ - "anyhow", - "aptos-config", - "aptos-crypto", - "aptos-db", - "aptos-executor", - "aptos-storage-interface", - "aptos-temppath", - "aptos-types", - "aptos-vm", - "bcs 0.1.4", - "clap 4.3.21", -] - [[package]] name = "aptos-db-indexer" version = "0.1.0" @@ -1126,6 +1109,7 @@ dependencies = [ "aptos-backup-service", "aptos-config", "aptos-db", + "aptos-executor", "aptos-executor-test-helpers", "aptos-executor-types", "aptos-logger", @@ -1134,7 +1118,9 @@ dependencies = [ "aptos-storage-interface", "aptos-temppath", "aptos-types", + "aptos-vm", "async-trait", + "bcs 0.1.4", "clap 4.3.21", "itertools 0.10.5", "owo-colors", @@ -1146,33 +1132,12 @@ name = "aptos-debugger" version = "0.1.0" dependencies = [ "anyhow", - "aptos-crypto", - "aptos-gas-meter", - "aptos-gas-profiling", - "aptos-gas-schedule", + "aptos-db-tool", "aptos-logger", - "aptos-memory-usage-tracker", - "aptos-resource-viewer", - "aptos-rest-client", - "aptos-state-view", - "aptos-table-natives", - "aptos-types", - "aptos-validator-interface", - "aptos-vm", - "aptos-vm-logging", - "aptos-vm-types", - "bcs 0.1.4", + "aptos-move-debugger", + "aptos-push-metrics", "clap 4.3.21", - "move-binary-format", - "move-cli", - "move-compiler", - "move-core-types", - "move-resource-viewer", - "move-vm-runtime", - "move-vm-test-utils", - "regex", "tokio", - "url", ] [[package]] @@ -2520,6 +2485,40 @@ dependencies = [ "prometheus", ] +[[package]] +name = "aptos-move-debugger" +version = "0.1.0" +dependencies = [ + "anyhow", + "aptos-crypto", + "aptos-gas-meter", + "aptos-gas-profiling", + "aptos-gas-schedule", + "aptos-logger", + "aptos-memory-usage-tracker", + "aptos-resource-viewer", + "aptos-rest-client", + "aptos-state-view", + "aptos-table-natives", + "aptos-types", + "aptos-validator-interface", + "aptos-vm", + "aptos-vm-logging", + "aptos-vm-types", + "bcs 0.1.4", + "clap 4.3.21", + "move-binary-format", + "move-cli", + "move-compiler", + "move-core-types", + "move-resource-viewer", + "move-vm-runtime", + "move-vm-test-utils", + "regex", + "tokio", + "url", +] + [[package]] name = "aptos-move-examples" version = "0.1.0" @@ -13684,7 +13683,6 @@ dependencies = [ "aptos-config", "aptos-consensus", "aptos-crypto", - "aptos-debugger", "aptos-faucet-core", "aptos-forge", "aptos-framework", @@ -13697,6 +13695,7 @@ dependencies = [ "aptos-inspection-service", "aptos-keygen", "aptos-logger", + "aptos-move-debugger", "aptos-release-builder", "aptos-rest-client", "aptos-rosetta", diff --git a/Cargo.toml b/Cargo.toml index f4d358a21db54..aba16ce7071f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ members = [ "crates/aptos-compression", "crates/aptos-crypto", "crates/aptos-crypto-derive", + "crates/aptos-debugger", "crates/aptos-drop-helper", "crates/aptos-enum-conversion-derive", "crates/aptos-faucet/cli", @@ -115,7 +116,6 @@ members = [ "ecosystem/node-checker", "ecosystem/node-checker/fn-check-client", "execution/block-partitioner", - "execution/db-bootstrapper", "execution/executor", "execution/executor-benchmark", "execution/executor-service", @@ -242,13 +242,13 @@ default-members = [ "aptos-node", "consensus/safety-rules", "crates/aptos", + "crates/aptos-debugger", "crates/aptos-faucet/service", "crates/aptos-keygen", "crates/aptos-rate-limiter", "crates/aptos-rosetta", "crates/transaction-emitter", "aptos-move/framework", - "execution/db-bootstrapper", "storage/backup/backup-cli", "ecosystem/node-checker", ] @@ -296,7 +296,7 @@ aptos-data-streaming-service = { path = "state-sync/data-streaming-service" } aptos-db = { path = "storage/aptosdb" } aptos-db-indexer = { path = "storage/indexer" } aptos-db-tool = { path = "storage/db-tool" } -aptos-debugger = { path = "aptos-move/aptos-debugger" } +aptos-debugger = { path = "crates/aptos-debugger" } aptos-drop-helper = { path = "crates/aptos-drop-helper" } aptos-event-notifications = { path = "state-sync/inter-component/event-notifications" } aptos-executable-store = { path = "storage/executable-store" } @@ -350,6 +350,7 @@ aptos-mempool = { path = "mempool" } aptos-mempool-notifications = { path = "state-sync/inter-component/mempool-notifications" } aptos-memsocket = { path = "network/memsocket" } aptos-metrics-core = { path = "crates/aptos-metrics-core" } +aptos-move-debugger = { path = "aptos-move/aptos-debugger" } aptos-move-examples = { path = "aptos-move/move-examples" } aptos-moving-average = { path = "crates/moving-average" } aptos-mvhashmap = { path = "aptos-move/mvhashmap" } @@ -741,10 +742,6 @@ debug-assertions = true [profile.bench] debug = true -# This is a temporary workaround to avoid multiple library -# definitions for LZ4 (caused by rust-rocksdb). -# This will be removed once our pull requests land. -# https://github.com/rust-rocksdb/rust-rocksdb/issues/666 [patch.crates-io] serde-reflection = { git = "https://github.com/aptos-labs/serde-reflection", rev = "839aed62a20ddccf043c08961cfe74875741ccba" } merlin = { git = "https://github.com/aptos-labs/merlin" } diff --git a/aptos-move/aptos-debugger/Cargo.toml b/aptos-move/aptos-debugger/Cargo.toml index 2b7ecb64c2e55..7a100a466ccca 100644 --- a/aptos-move/aptos-debugger/Cargo.toml +++ b/aptos-move/aptos-debugger/Cargo.toml @@ -1,14 +1,16 @@ [package] -name = "aptos-debugger" +name = "aptos-move-debugger" version = "0.1.0" -authors = ["Aptos Labs "] -description = "A tool to replay transactions on chain and to execute transactions locally with remote backend" -repository = "https://github.com/aptos-labs/aptos-core" -homepage = "https://aptoslabs.com" -license = "Apache-2.0" -publish = false -edition = "2021" -default-run = "aptos-debugger" +description = "A tool to replay transactions on chain and to execute transactions locally." + +# Workspace inherited keys +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } [dependencies] anyhow = { workspace = true } @@ -39,6 +41,3 @@ move-vm-test-utils = { workspace = true } regex = { workspace = true } tokio = { workspace = true } url = { workspace = true } - -[[bin]] -name = "bcs-txn-decoder" diff --git a/aptos-move/aptos-debugger/README.md b/aptos-move/aptos-debugger/README.md deleted file mode 100644 index 8b40af9bb1948..0000000000000 --- a/aptos-move/aptos-debugger/README.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -id: aptos-debugger -title: Aptos Debugger -custom_edit_url: https://github.com/aptos-labs/aptos-core/edit/main/language/aptos-debugger/README.md ---- - -# Aptos Debugger - -``` -$ cargo run -p aptos-debugger - -Usage: aptos-debugger [OPTIONS] --begin-version --limit - -Commands: - rest Use full node's rest api as query endpoint - db Use a local db instance to serve as query endpoint - help Print this message or the help of the given subcommand(s) - -Options: - --begin-version - --limit - --concurrency-level [default: 1] - -h, --help Print help -``` - -# BCS transaction decoder - -To use, run: -``` -$ cargo run -p aptos-debugger --bin bcs-txn-decoder -``` -Once it starts, it waits for a standard input. So please copy and paste a BCS-encoded transaction like: -``` -[65, 2, 62, 205, 111, 255, 182, 248, 10, 156, 99, 251, 214, 36, 49, 107, 225, 63, 1, 93, 30, 222, 207, 61, 178, 136, 229, 64, 145, 48, 55, 131, 4, 1, 0, 0, 0, 0, 0, 0, 2, 253, 249, 242, 150, 39, 16, 224, 114, 46, 38, 148, 6, 20, 25, 220, 21, 148, 64, 90, 94, 71, 138, 29, 35, 35, 80, 6, 154, 50, 83, 255, 148, 7, 119, 97, 114, 107, 97, 100, 101, 12, 109, 105, 110, 116, 95, 119, 97, 114, 107, 97, 100, 101, 0, 0, 160, 134, 1, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 190, 30, 118, 100, 0, 0, 0, 0, 2, 0, 32, 243, 52, 118, 7, 1, 74, 17, 119, 172, 46, 175, 211, 29, 153, 62, 59, 25, 207, 198, 210, 25, 238, 108, 2, 67, 153, 41, 80, 108, 107, 247, 63, 64, 69, 72, 73, 1, 249, 2, 60, 227, 143, 119, 179, 185, 250, 22, 94, 11, 27, 241, 3, 223, 177, 205, 77, 207, 121, 199, 94, 239, 117, 96, 102, 42, 169, 227, 80, 250, 182, 132, 103, 45, 174, 196, 87, 101, 7, 178, 68, 153, 241, 207, 83, 160, 220, 4, 0, 19, 38, 244, 109, 43, 113, 212, 12, 9] -``` - -Note: Your terminal may suffer from pasting a large amount data from the clipboard. In that case, save the transaction data as a text file and use the redirection as follows: -``` -$ cargo run -p aptos-debugger --bin bcs-txn-decoder < example-txn.txt -``` - -Then, it will decode the transaction, and output the following information: -``` -=================== -Transaction Summary -=================== -Sender: 41023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e54091303783 -Sequence number: 260 -Chain ID: 2 -Network: testnet -Version: 531155369 -Overview: https://explorer.aptoslabs.com/txn/531155369/userTxnOverview?network=testnet -Payload: https://explorer.aptoslabs.com/txn/531155369/payload?network=testnet -``` - -To decode and execute the transaction automatically, use the `--execute` flag like: -``` -$ cargo run -p aptos-debugger --bin bcs-txn-decoder -- --execute -``` - -You will get the following message additionally: -``` -=============================== -Transaction re-execution result -=============================== -Got 1/1 txns from RestApi. -Starting epoch execution at 531155369, 1 transactions remaining -2023-07-13T20:10:02.356255Z [main] INFO aptos-move/aptos-vm/src/aptos_vm.rs:1505 Executing block, transaction count: 1 {"name":"miscellaneous","txn_idx":0} -2023-07-13T20:10:05.784004Z [main] INFO aptos-move/block-executor/src/executor.rs:794 [Execution]: Sequential execution completed, all 1 txns committed. -[ - TransactionOutput { - write_set: V0( - WriteSetV0( - WriteSetMut { - write_set: { - StateKey { - inner: AccessPath( - AccessPath { address: 07f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb, path: 010000000000000000000000000000000000000000000000000000000000000001076163636f756e74074163636f756e7400 }, - ), - hash: OnceCell(Uninit), - }: Modification(200000000000000000000000000000000000000000000000000000000000000000000000000000000047260000000000000000000000000000000000000000000007f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb0000000000000000010000000000000007f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb000107f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb), - StateKey { - inner: AccessPath( - AccessPath { address: 41023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e54091303783, path: 01000000000000000000000000000000000000000000000000000000000000000104636f696e09436f696e53746f7265010700000000000000000000000000000000000000000000000000000000000000010a6170746f735f636f696e094170746f73436f696e00 }, - ), - hash: OnceCell(Uninit), - }: Modification(cf3c904a00000000000600000000000000020000000000000041023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e540913037830501000000000000030000000000000041023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e54091303783), - StateKey { - inner: AccessPath( - AccessPath { address: 41023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e54091303783, path: 010000000000000000000000000000000000000000000000000000000000000001076163636f756e74074163636f756e7400 }, - ), - hash: OnceCell(Uninit), - }: Modification(2041023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e54091303783050100000000000004000000000000000100000000000000000000000000000041023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e540913037830000000000000000010000000000000041023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e540913037830000), - StateKey { - inner: AccessPath( - AccessPath { address: db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d, path: 020000000000000000000000000000000000000000000000000000000000000001066f626a6563740b4f626a65637447726f757000 }, - ), - hash: OnceCell(Uninit), - }: Creation(040000000000000000000000000000000000000000000000000000000000000001066f626a6563740a4f626a656374436f72650059020000000000040041023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e540913037830101000000000000000000000000000400db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d000000000000000000000000000000000000000000000000000000000000000405746f6b656e05546f6b656e008403e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c24526000000000000e4014170746f73205761726361646520697320612066756e20616e6420656e676167696e67204e46542062617365642067616d65706c6179206f6e204170746f732e2045766572792074696d6520612075736572206d696e74732c206120756e69717565204e46542069732067656e657261746564206261736564206f6e2072616e646f6d6e65737320776974682070726f626162696c6974696573206f66206d696e74696e6720576172636164657320616e6420204461726b204c6f72642e2057686f65766572206d696e747320746865204461726b204c6f7264206b696c6c732069742e0d576172636164652023393739373768747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f77617263616469616275636b65742f393739372e6a736f6e00000000000000000100000000000400db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d00000000000000000000000000000000000000000000000000000000000000040b6170746f735f746f6b656e0a4170746f73546f6b656e0085010101db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d0001db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d01db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287ddb965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d00000000000000000000000000000000000000000000000000000000000000040c70726f70657274795f6d61700b50726f70657274794d6170004e070b4163636573736f7269657301010206576561706f6e0101060448616e640101030848656164676561720101050641726d6f757201010804426f64790101050a4261636b67726f756e64010108), - StateKey { - inner: AccessPath( - AccessPath { address: e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c2, path: 020000000000000000000000000000000000000000000000000000000000000001066f626a6563740b4f626a65637447726f757000 }, - ), - hash: OnceCell(Uninit), - }: Modification(050000000000000000000000000000000000000000000000000000000000000001066f626a6563740a4f626a656374436f72650059040000000000040007f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb0000000000000000000000000000000400e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c2000000000000000000000000000000000000000000000000000000000000000407726f79616c747907526f79616c747900300400000000000000640000000000000007f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb00000000000000000000000000000000000000000000000000000000000000040a636f6c6c656374696f6e0a436f6c6c656374696f6e00ed0207f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcbe4014170746f73205761726361646520697320612066756e20616e6420656e676167696e67204e46542062617365642067616d65706c6179206f6e204170746f732e2045766572792074696d6520612075736572206d696e74732c206120756e69717565204e46542069732067656e657261746564206261736564206f6e2072616e646f6d6e65737320776974682070726f626162696c6974696573206f66206d696e74696e6720576172636164657320616e6420204461726b204c6f72642e2057686f65766572206d696e747320746865204461726b204c6f7264206b696c6c732069742e07576172636164652e68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f77617263616469616275636b65742f00000000000000000300000000000400e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c200000000000000000000000000000000000000000000000000000000000000040a636f6c6c656374696f6e0b4669786564537570706c790078452600000000000000ca9a3b00000000452600000000000000000000000000000100000000000400e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c245260000000000000200000000000400e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c200000000000000000000000000000000000000000000000000000000000000040b6170746f735f746f6b656e0f4170746f73436f6c6c656374696f6e004a01e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c201e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c20101010101010101), - StateKey { - inner: AccessPath( - AccessPath { address: fdf9f2962710e0722e2694061419dc1594405a5e478a1d232350069a3253ff94, path: 01000000000000000000000000000000000000000000000000000000000000000104636f696e09436f696e53746f7265010700000000000000000000000000000000000000000000000000000000000000010a6170746f735f636f696e094170746f73436f696e00 }, - ), - hash: OnceCell(Uninit), - }: Modification(0cede205000000000046260000000000000200000000000000fdf9f2962710e0722e2694061419dc1594405a5e478a1d232350069a3253ff9409000000000000000300000000000000fdf9f2962710e0722e2694061419dc1594405a5e478a1d232350069a3253ff94), - StateKey { - inner: AccessPath( - AccessPath { address: fdf9f2962710e0722e2694061419dc1594405a5e478a1d232350069a3253ff94, path: 01fdf9f2962710e0722e2694061419dc1594405a5e478a1d232350069a3253ff94077761726b616465084d696e74496e666f00 }, - ), - hash: OnceCell(Uninit), - }: Modification(2e68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f77617263616469616275636b65742f4526000000000000070b000000000000000b000000000000000b000000000000000b000000000000000b000000000000000b000000000000000b0000000000000007f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb07f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb00000000000000000757617263616465e4014170746f73205761726361646520697320612066756e20616e6420656e676167696e67204e46542062617365642067616d65706c6179206f6e204170746f732e2045766572792074696d6520612075736572206d696e74732c206120756e69717565204e46542069732067656e657261746564206261736564206f6e2072616e646f6d6e65737320776974682070726f626162696c6974696573206f66206d696e74696e6720576172636164657320616e6420204461726b204c6f72642e2057686f65766572206d696e747320746865204461726b204c6f7264206b696c6c732069742e070a4261636b67726f756e6404426f64790641726d6f75720848656164676561720448616e6406576561706f6e0b4163636573736f7269657307027538027538027538027538027538027538027538), - StateKey { - inner: TableItem { - handle: TableHandle( - 1b854694ae746cdbd8d44186ca4929b2b337df21d1c74633be19b2710552fdca, - ), - key: [ - 6, - 25, - 220, - 41, - 160, - 170, - 200, - 250, - 20, - 103, - 20, - 5, - 142, - 141, - 214, - 210, - 208, - 243, - 189, - 245, - 246, - 51, - 25, - 7, - 191, - 145, - 243, - 172, - 216, - 30, - 105, - 53, - ], - }, - hash: OnceCell(Uninit), - }: Modification(fddc5282534cd0010100000000000000), - }, - }, - ), - ), - events: [ - ContractEvent { key: EventKey { creation_number: 1125899906842626, account_address: e1082d4b1e45fd72ee5d9057dc7817533cbce491497c1c5eea9e658fda1060c2 }, index: 9796, type: Struct(StructTag { address: 0000000000000000000000000000000000000000000000000000000000000004, module: Identifier("collection"), name: Identifier("MintEvent"), type_params: [] }), event_data: "4526000000000000db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d" }, - ContractEvent { key: EventKey { creation_number: 1125899906842624, account_address: db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d }, index: 0, type: Struct(StructTag { address: 0000000000000000000000000000000000000000000000000000000000000001, module: Identifier("object"), name: Identifier("TransferEvent"), type_params: [] }), event_data: "db965d3a21e181802e0518be1d68dc7449b7cb335cfd97978531ae5b7f33287d07f218c4fe83285086ae6a929b67cff11cfd896a93e7dade2bd19f13404e7bcb41023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e54091303783" }, - ContractEvent { key: EventKey { creation_number: 3, account_address: 41023ecd6fffb6f80a9c63fbd624316be13f015d1edecf3db288e54091303783 }, index: 260, type: Struct(StructTag { address: 0000000000000000000000000000000000000000000000000000000000000001, module: Identifier("coin"), name: Identifier("WithdrawEvent"), type_params: [] }), event_data: "0000000000000000" }, - ContractEvent { key: EventKey { creation_number: 2, account_address: fdf9f2962710e0722e2694061419dc1594405a5e478a1d232350069a3253ff94 }, index: 9797, type: Struct(StructTag { address: 0000000000000000000000000000000000000000000000000000000000000001, module: Identifier("coin"), name: Identifier("DepositEvent"), type_params: [] }), event_data: "0000000000000000" }, - ], - gas_used: 586, - status: Keep( - Success, - ), - }, -] -``` diff --git a/aptos-move/aptos-debugger/src/aptos_debugger.rs b/aptos-move/aptos-debugger/src/aptos_debugger.rs new file mode 100644 index 0000000000000..4c29e5a926df9 --- /dev/null +++ b/aptos-move/aptos-debugger/src/aptos_debugger.rs @@ -0,0 +1,318 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{format_err, Result}; +use aptos_gas_meter::{StandardGasAlgebra, StandardGasMeter}; +use aptos_gas_profiling::{GasProfiler, TransactionGasLog}; +use aptos_gas_schedule::{MiscGasParameters, NativeGasParameters, LATEST_GAS_FEATURE_VERSION}; +use aptos_memory_usage_tracker::MemoryTrackedGasMeter; +use aptos_resource_viewer::{AnnotatedAccountStateBlob, AptosValueAnnotator}; +use aptos_rest_client::Client; +use aptos_state_view::TStateView; +use aptos_types::{ + account_address::AccountAddress, + chain_id::ChainId, + on_chain_config::{Features, OnChainConfig, TimedFeaturesBuilder}, + transaction::{ + signature_verified_transaction::SignatureVerifiedTransaction, SignedTransaction, + Transaction, TransactionInfo, TransactionOutput, TransactionPayload, Version, + }, + vm_status::VMStatus, +}; +use aptos_validator_interface::{ + AptosValidatorInterface, DBDebuggerInterface, DebuggerStateView, RestDebuggerInterface, +}; +use aptos_vm::{ + data_cache::AsMoveResolver, + move_vm_ext::{MoveVmExt, SessionExt, SessionId}, + AptosVM, VMExecutor, +}; +use aptos_vm_logging::log_schema::AdapterLogSchema; +use aptos_vm_types::{change_set::VMChangeSet, output::VMOutput, storage::ChangeSetConfigs}; +use clap::Parser; +use move_binary_format::errors::VMResult; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; +use url::Url; + +#[derive(Parser)] +#[clap(group(clap::ArgGroup::new("target") + .required(true) + .args(&["rest_endpoint", "db_path"]), +))] +pub struct Command { + /// Use full node's rest api as query endpoint. + #[clap(long, group = "target")] + rest_endpoint: Option, + + /// Use a local db instance to serve as query endpoint. + #[clap(long, group = "target")] + db_path: Option, + + #[clap(long)] + begin_version: u64, + + #[clap(long)] + limit: u64, + + #[clap(long, default_value_t = 1)] + concurrency_level: usize, +} + +impl Command { + pub async fn run(self) -> Result<()> { + AptosVM::set_concurrency_level_once(self.concurrency_level); + + let debugger = if let Some(rest_endpoint) = self.rest_endpoint { + AptosDebugger::rest_client(Client::new(Url::parse(&rest_endpoint)?))? + } else { + AptosDebugger::db(self.db_path.unwrap())? + }; + + println!( + "{:#?}", + debugger + .execute_past_transactions(self.begin_version, self.limit) + .await? + ); + + Ok(()) + } +} + +pub struct AptosDebugger { + debugger: Arc, +} + +impl AptosDebugger { + pub fn new(debugger: Arc) -> Self { + Self { debugger } + } + + pub fn rest_client(rest_client: Client) -> Result { + Ok(Self::new(Arc::new(RestDebuggerInterface::new(rest_client)))) + } + + pub fn db + Clone>(db_root_path: P) -> Result { + Ok(Self::new(Arc::new(DBDebuggerInterface::open( + db_root_path, + )?))) + } + + pub fn execute_transactions_at_version( + &self, + version: Version, + txns: Vec, + ) -> Result> { + let sig_verified_txns: Vec = + txns.into_iter().map(|x| x.into()).collect::>(); + let state_view = DebuggerStateView::new(self.debugger.clone(), version); + AptosVM::execute_block(&sig_verified_txns, &state_view, None) + .map_err(|err| format_err!("Unexpected VM Error: {:?}", err)) + } + + pub fn execute_transaction_at_version_with_gas_profiler( + &self, + version: Version, + txn: SignedTransaction, + ) -> Result<(VMStatus, VMOutput, TransactionGasLog)> { + let state_view = DebuggerStateView::new(self.debugger.clone(), version); + let log_context = AdapterLogSchema::new(state_view.id(), 0); + let txn = txn + .check_signature() + .map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; + + // TODO(Gas): revisit this. + let vm = AptosVM::new_from_state_view(&state_view); + let resolver = state_view.as_move_resolver(); + + let (status, output, gas_profiler) = vm.execute_user_transaction_with_custom_gas_meter( + &resolver, + &txn, + &log_context, + |gas_feature_version, gas_params, storage_gas_params, balance| { + let gas_meter = + MemoryTrackedGasMeter::new(StandardGasMeter::new(StandardGasAlgebra::new( + gas_feature_version, + gas_params, + storage_gas_params, + balance, + ))); + let gas_profiler = match txn.payload() { + TransactionPayload::Script(_) => GasProfiler::new_script(gas_meter), + TransactionPayload::EntryFunction(entry_func) => GasProfiler::new_function( + gas_meter, + entry_func.module().clone(), + entry_func.function().to_owned(), + entry_func.ty_args().to_vec(), + ), + TransactionPayload::ModuleBundle(..) => unreachable!("not supported"), + TransactionPayload::Multisig(..) => unimplemented!("not supported yet"), + }; + Ok(gas_profiler) + }, + )?; + + Ok((status, output, gas_profiler.finish())) + } + + pub async fn execute_past_transactions( + &self, + mut begin: Version, + mut limit: u64, + ) -> Result> { + let (mut txns, mut txn_infos) = self + .debugger + .get_committed_transactions(begin, limit) + .await?; + + let mut ret = vec![]; + while limit != 0 { + println!( + "Starting epoch execution at {:?}, {:?} transactions remaining", + begin, limit + ); + let mut epoch_result = self + .execute_transactions_by_epoch(begin, txns.clone()) + .await?; + begin += epoch_result.len() as u64; + limit -= epoch_result.len() as u64; + txns = txns.split_off(epoch_result.len()); + let epoch_txn_infos = txn_infos.drain(0..epoch_result.len()).collect::>(); + Self::print_mismatches(&epoch_result, &epoch_txn_infos, begin); + + ret.append(&mut epoch_result); + } + Ok(ret) + } + + fn print_mismatches( + txn_outputs: &[TransactionOutput], + expected_txn_infos: &[TransactionInfo], + first_version: Version, + ) { + for idx in 0..txn_outputs.len() { + let txn_output = &txn_outputs[idx]; + let txn_info = &expected_txn_infos[idx]; + let version = first_version + idx as Version; + txn_output + .ensure_match_transaction_info(version, txn_info, None, None) + .unwrap_or_else(|err| println!("{}", err)) + } + } + + pub async fn execute_transactions_by_epoch( + &self, + begin: Version, + txns: Vec, + ) -> Result> { + let results = self.execute_transactions_at_version(begin, txns)?; + let mut ret = vec![]; + let mut is_reconfig = false; + + for result in results.into_iter() { + if is_reconfig { + continue; + } + if is_reconfiguration(&result) { + is_reconfig = true; + } + ret.push(result) + } + Ok(ret) + } + + pub async fn annotate_account_state_at_version( + &self, + account: AccountAddress, + version: Version, + ) -> Result> { + let state_view = DebuggerStateView::new(self.debugger.clone(), version); + let remote_storage = state_view.as_move_resolver(); + let annotator = AptosValueAnnotator::new(&remote_storage); + Ok( + match self + .debugger + .get_account_state_by_version(account, version) + .await? + { + Some(account_state) => Some(annotator.view_account_state(&account_state)?), + None => None, + }, + ) + } + + pub async fn annotate_key_accounts_at_version( + &self, + version: Version, + ) -> Result> { + let accounts = self.debugger.get_admin_accounts(version).await?; + let state_view = DebuggerStateView::new(self.debugger.clone(), version); + let remote_storage = state_view.as_move_resolver(); + let annotator = AptosValueAnnotator::new(&remote_storage); + + let mut result = vec![]; + for (addr, state) in accounts.into_iter() { + result.push((addr, annotator.view_account_state(&state)?)); + } + Ok(result) + } + + pub async fn get_latest_version(&self) -> Result { + self.debugger.get_latest_version().await + } + + pub async fn get_version_by_account_sequence( + &self, + account: AccountAddress, + seq: u64, + ) -> Result> { + self.debugger + .get_version_by_account_sequence(account, seq) + .await + } + + pub fn run_session_at_version(&self, version: Version, f: F) -> Result + where + F: FnOnce(&mut SessionExt) -> VMResult<()>, + { + let state_view = DebuggerStateView::new(self.debugger.clone(), version); + let state_view_storage = state_view.as_move_resolver(); + let features = Features::fetch_config(&state_view_storage).unwrap_or_default(); + let move_vm = MoveVmExt::new( + NativeGasParameters::zeros(), + MiscGasParameters::zeros(), + LATEST_GAS_FEATURE_VERSION, + ChainId::test().id(), + features, + TimedFeaturesBuilder::enable_all().build(), + &state_view_storage, + ) + .unwrap(); + let mut session = move_vm.new_session(&state_view_storage, SessionId::Void); + f(&mut session).map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; + let change_set = session + .finish( + &mut (), + &ChangeSetConfigs::unlimited_at_gas_feature_version(LATEST_GAS_FEATURE_VERSION), + ) + .map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; + Ok(change_set) + } +} + +fn is_reconfiguration(vm_output: &TransactionOutput) -> bool { + let new_epoch_event_key = aptos_types::on_chain_config::new_epoch_event_key(); + vm_output + .events() + .iter() + .any(|event| event.event_key() == Some(&new_epoch_event_key)) +} + +#[test] +fn verify_tool() { + use clap::CommandFactory; + Command::command().debug_assert() +} diff --git a/aptos-move/aptos-debugger/src/bcs_txn_decoder.rs b/aptos-move/aptos-debugger/src/bcs_txn_decoder.rs new file mode 100644 index 0000000000000..9a4cc2a76051f --- /dev/null +++ b/aptos-move/aptos-debugger/src/bcs_txn_decoder.rs @@ -0,0 +1,81 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use crate::aptos_debugger::AptosDebugger; +use anyhow::Result; +use aptos_rest_client::Client; +use aptos_types::transaction::SignedTransaction; +use aptos_vm::AptosVM; +use clap::Parser; +use regex::Regex; +use std::io; +use url::Url; + +#[derive(Parser)] +pub struct Command { + #[clap(long, default_value_t = false)] + execute: bool, + + #[clap(long, default_value_t = 1)] + concurrency_level: usize, +} + +impl Command { + pub async fn run(self) -> Result<()> { + let mut buffer = String::new(); + io::stdin().read_line(&mut buffer)?; + let re = Regex::new(r"\d+").unwrap(); + let bytes = re + .find_iter(&buffer) + .filter_map(|m| m.as_str().parse::().ok()) + .collect::>(); + + let txn: SignedTransaction = bcs::from_bytes::(&bytes)?; + let chain_id = txn.chain_id(); + println!("==================="); + println!("Transaction Summary"); + println!("==================="); + println!("Sender: {:?}", txn.sender()); + println!("Sequence number: {:?}", txn.sequence_number()); + + let network = if chain_id.is_mainnet() { + "mainnet".to_string() + } else if chain_id.is_testnet() { + "testnet".to_string() + } else { + "devnet".to_string() + }; + println!("Chain ID: {}", chain_id.id()); + println!("Network: {}", network); + + let endpoint = format!("https://{}.aptoslabs.com/v1", network); + let debugger = AptosDebugger::rest_client(Client::new(Url::parse(&endpoint)?))?; + let version = debugger + .get_version_by_account_sequence(txn.sender(), txn.sequence_number()) + .await? + .unwrap(); + println!("Version: {:?}", version); + println!( + "Overview: https://explorer.aptoslabs.com/txn/{:?}/userTxnOverview?network={}", + version, network + ); + println!( + "Payload: https://explorer.aptoslabs.com/txn/{:?}/payload?network={}", + version, network + ); + + if self.execute { + AptosVM::set_concurrency_level_once(self.concurrency_level); + println!(); + println!("==============================="); + println!("Transaction re-execution result"); + println!("==============================="); + println!( + "{:#?}", + debugger.execute_past_transactions(version, 1).await? + ); + } + + Ok(()) + } +} diff --git a/aptos-move/aptos-debugger/src/bin/bcs-txn-decoder.rs b/aptos-move/aptos-debugger/src/bin/bcs-txn-decoder.rs deleted file mode 100644 index 2ac18477f04a0..0000000000000 --- a/aptos-move/aptos-debugger/src/bin/bcs-txn-decoder.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::Result; -use aptos_debugger::AptosDebugger; -use aptos_rest_client::Client; -use aptos_types::transaction::SignedTransaction; -use aptos_vm::AptosVM; -use clap::Parser; -use regex::Regex; -use std::io; -use url::Url; - -#[derive(Parser)] -pub struct Argument { - #[clap(long, default_value_t = false)] - execute: bool, - - #[clap(long, default_value_t = 1)] - concurrency_level: usize, -} - -#[tokio::main] -async fn main() -> Result<()> { - let mut buffer = String::new(); - io::stdin().read_line(&mut buffer)?; - let re = Regex::new(r"\d+").unwrap(); - let bytes = re - .find_iter(&buffer) - .filter_map(|m| m.as_str().parse::().ok()) - .collect::>(); - - let txn: SignedTransaction = bcs::from_bytes::(&bytes)?; - let chain_id = txn.chain_id(); - println!("==================="); - println!("Transaction Summary"); - println!("==================="); - println!("Sender: {:?}", txn.sender()); - println!("Sequence number: {:?}", txn.sequence_number()); - - let network = if chain_id.is_mainnet() { - "mainnet".to_string() - } else if chain_id.is_testnet() { - "testnet".to_string() - } else { - "devnet".to_string() - }; - println!("Chain ID: {}", chain_id.id()); - println!("Network: {}", network); - - let endpoint = format!("https://{}.aptoslabs.com/v1", network); - let debugger = AptosDebugger::rest_client(Client::new(Url::parse(&endpoint)?))?; - let version = debugger - .get_version_by_account_sequence(txn.sender(), txn.sequence_number()) - .await? - .unwrap(); - println!("Version: {:?}", version); - println!( - "Overview: https://explorer.aptoslabs.com/txn/{:?}/userTxnOverview?network={}", - version, network - ); - println!( - "Payload: https://explorer.aptoslabs.com/txn/{:?}/payload?network={}", - version, network - ); - - let args = Argument::parse(); - if args.execute { - aptos_logger::Logger::new().init(); - AptosVM::set_concurrency_level_once(args.concurrency_level); - println!(); - println!("==============================="); - println!("Transaction re-execution result"); - println!("==============================="); - println!( - "{:#?}", - debugger.execute_past_transactions(version, 1).await? - ); - } - - Ok(()) -} diff --git a/aptos-move/aptos-debugger/src/lib.rs b/aptos-move/aptos-debugger/src/lib.rs index 05554764da23c..79c9908a89568 100644 --- a/aptos-move/aptos-debugger/src/lib.rs +++ b/aptos-move/aptos-debugger/src/lib.rs @@ -1,262 +1,5 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use anyhow::{format_err, Result}; -use aptos_gas_meter::{StandardGasAlgebra, StandardGasMeter}; -use aptos_gas_profiling::{GasProfiler, TransactionGasLog}; -use aptos_gas_schedule::{MiscGasParameters, NativeGasParameters, LATEST_GAS_FEATURE_VERSION}; -use aptos_memory_usage_tracker::MemoryTrackedGasMeter; -use aptos_resource_viewer::{AnnotatedAccountStateBlob, AptosValueAnnotator}; -use aptos_rest_client::Client; -use aptos_state_view::TStateView; -use aptos_types::{ - account_address::AccountAddress, - chain_id::ChainId, - on_chain_config::{Features, OnChainConfig, TimedFeaturesBuilder}, - transaction::{ - signature_verified_transaction::SignatureVerifiedTransaction, SignedTransaction, - Transaction, TransactionInfo, TransactionOutput, TransactionPayload, Version, - }, - vm_status::VMStatus, -}; -use aptos_validator_interface::{ - AptosValidatorInterface, DBDebuggerInterface, DebuggerStateView, RestDebuggerInterface, -}; -use aptos_vm::{ - data_cache::AsMoveResolver, - move_vm_ext::{MoveVmExt, SessionExt, SessionId}, - AptosVM, VMExecutor, -}; -use aptos_vm_logging::log_schema::AdapterLogSchema; -use aptos_vm_types::{change_set::VMChangeSet, output::VMOutput, storage::ChangeSetConfigs}; -use move_binary_format::errors::VMResult; -use std::{path::Path, sync::Arc}; - -pub struct AptosDebugger { - debugger: Arc, -} - -impl AptosDebugger { - pub fn new(debugger: Arc) -> Self { - Self { debugger } - } - - pub fn rest_client(rest_client: Client) -> Result { - Ok(Self::new(Arc::new(RestDebuggerInterface::new(rest_client)))) - } - - pub fn db + Clone>(db_root_path: P) -> Result { - Ok(Self::new(Arc::new(DBDebuggerInterface::open( - db_root_path, - )?))) - } - - pub fn execute_transactions_at_version( - &self, - version: Version, - txns: Vec, - ) -> Result> { - let sig_verified_txns: Vec = - txns.into_iter().map(|x| x.into()).collect::>(); - let state_view = DebuggerStateView::new(self.debugger.clone(), version); - AptosVM::execute_block(&sig_verified_txns, &state_view, None) - .map_err(|err| format_err!("Unexpected VM Error: {:?}", err)) - } - - pub fn execute_transaction_at_version_with_gas_profiler( - &self, - version: Version, - txn: SignedTransaction, - ) -> Result<(VMStatus, VMOutput, TransactionGasLog)> { - let state_view = DebuggerStateView::new(self.debugger.clone(), version); - let log_context = AdapterLogSchema::new(state_view.id(), 0); - let txn = txn - .check_signature() - .map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; - - // TODO(Gas): revisit this. - let vm = AptosVM::new_from_state_view(&state_view); - let resolver = state_view.as_move_resolver(); - - let (status, output, gas_profiler) = vm.execute_user_transaction_with_custom_gas_meter( - &resolver, - &txn, - &log_context, - |gas_feature_version, gas_params, storage_gas_params, balance| { - let gas_meter = - MemoryTrackedGasMeter::new(StandardGasMeter::new(StandardGasAlgebra::new( - gas_feature_version, - gas_params, - storage_gas_params, - balance, - ))); - let gas_profiler = match txn.payload() { - TransactionPayload::Script(_) => GasProfiler::new_script(gas_meter), - TransactionPayload::EntryFunction(entry_func) => GasProfiler::new_function( - gas_meter, - entry_func.module().clone(), - entry_func.function().to_owned(), - entry_func.ty_args().to_vec(), - ), - TransactionPayload::ModuleBundle(..) => unreachable!("not supported"), - TransactionPayload::Multisig(..) => unimplemented!("not supported yet"), - }; - Ok(gas_profiler) - }, - )?; - - Ok((status, output, gas_profiler.finish())) - } - - pub async fn execute_past_transactions( - &self, - mut begin: Version, - mut limit: u64, - ) -> Result> { - let (mut txns, mut txn_infos) = self - .debugger - .get_committed_transactions(begin, limit) - .await?; - - let mut ret = vec![]; - while limit != 0 { - println!( - "Starting epoch execution at {:?}, {:?} transactions remaining", - begin, limit - ); - let mut epoch_result = self - .execute_transactions_by_epoch(begin, txns.clone()) - .await?; - begin += epoch_result.len() as u64; - limit -= epoch_result.len() as u64; - txns = txns.split_off(epoch_result.len()); - let epoch_txn_infos = txn_infos.drain(0..epoch_result.len()).collect::>(); - Self::print_mismatches(&epoch_result, &epoch_txn_infos, begin); - - ret.append(&mut epoch_result); - } - Ok(ret) - } - - fn print_mismatches( - txn_outputs: &[TransactionOutput], - expected_txn_infos: &[TransactionInfo], - first_version: Version, - ) { - for idx in 0..txn_outputs.len() { - let txn_output = &txn_outputs[idx]; - let txn_info = &expected_txn_infos[idx]; - let version = first_version + idx as Version; - txn_output - .ensure_match_transaction_info(version, txn_info, None, None) - .unwrap_or_else(|err| println!("{}", err)) - } - } - - pub async fn execute_transactions_by_epoch( - &self, - begin: Version, - txns: Vec, - ) -> Result> { - let results = self.execute_transactions_at_version(begin, txns)?; - let mut ret = vec![]; - let mut is_reconfig = false; - - for result in results.into_iter() { - if is_reconfig { - continue; - } - if is_reconfiguration(&result) { - is_reconfig = true; - } - ret.push(result) - } - Ok(ret) - } - - pub async fn annotate_account_state_at_version( - &self, - account: AccountAddress, - version: Version, - ) -> Result> { - let state_view = DebuggerStateView::new(self.debugger.clone(), version); - let remote_storage = state_view.as_move_resolver(); - let annotator = AptosValueAnnotator::new(&remote_storage); - Ok( - match self - .debugger - .get_account_state_by_version(account, version) - .await? - { - Some(account_state) => Some(annotator.view_account_state(&account_state)?), - None => None, - }, - ) - } - - pub async fn annotate_key_accounts_at_version( - &self, - version: Version, - ) -> Result> { - let accounts = self.debugger.get_admin_accounts(version).await?; - let state_view = DebuggerStateView::new(self.debugger.clone(), version); - let remote_storage = state_view.as_move_resolver(); - let annotator = AptosValueAnnotator::new(&remote_storage); - - let mut result = vec![]; - for (addr, state) in accounts.into_iter() { - result.push((addr, annotator.view_account_state(&state)?)); - } - Ok(result) - } - - pub async fn get_latest_version(&self) -> Result { - self.debugger.get_latest_version().await - } - - pub async fn get_version_by_account_sequence( - &self, - account: AccountAddress, - seq: u64, - ) -> Result> { - self.debugger - .get_version_by_account_sequence(account, seq) - .await - } - - pub fn run_session_at_version(&self, version: Version, f: F) -> Result - where - F: FnOnce(&mut SessionExt) -> VMResult<()>, - { - let state_view = DebuggerStateView::new(self.debugger.clone(), version); - let state_view_storage = state_view.as_move_resolver(); - let features = Features::fetch_config(&state_view_storage).unwrap_or_default(); - let move_vm = MoveVmExt::new( - NativeGasParameters::zeros(), - MiscGasParameters::zeros(), - LATEST_GAS_FEATURE_VERSION, - ChainId::test().id(), - features, - TimedFeaturesBuilder::enable_all().build(), - &state_view_storage, - ) - .unwrap(); - let mut session = move_vm.new_session(&state_view_storage, SessionId::Void); - f(&mut session).map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; - let change_set = session - .finish( - &mut (), - &ChangeSetConfigs::unlimited_at_gas_feature_version(LATEST_GAS_FEATURE_VERSION), - ) - .map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; - Ok(change_set) - } -} - -fn is_reconfiguration(vm_output: &TransactionOutput) -> bool { - let new_epoch_event_key = aptos_types::on_chain_config::new_epoch_event_key(); - vm_output - .events() - .iter() - .any(|event| event.event_key() == Some(&new_epoch_event_key)) -} +pub mod aptos_debugger; +pub mod bcs_txn_decoder; diff --git a/aptos-move/aptos-debugger/src/main.rs b/aptos-move/aptos-debugger/src/main.rs deleted file mode 100644 index 9056c92359859..0000000000000 --- a/aptos-move/aptos-debugger/src/main.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::Result; -use aptos_debugger::AptosDebugger; -use aptos_rest_client::Client; -use aptos_vm::AptosVM; -use clap::{Parser, Subcommand}; -use std::path::PathBuf; -use url::Url; - -#[derive(Subcommand)] -pub enum Target { - /// Use full node's rest api as query endpoint. - Rest { endpoint: String }, - /// Use a local db instance to serve as query endpoint. - DB { path: PathBuf }, -} -#[derive(Parser)] -pub struct Argument { - #[clap(subcommand)] - target: Target, - - #[clap(long)] - begin_version: u64, - - #[clap(long)] - limit: u64, - - #[clap(long, default_value_t = 1)] - concurrency_level: usize, -} - -#[tokio::main] -async fn main() -> Result<()> { - aptos_logger::Logger::new().init(); - let args = Argument::parse(); - AptosVM::set_concurrency_level_once(args.concurrency_level); - - let debugger = match args.target { - Target::Rest { endpoint } => { - AptosDebugger::rest_client(Client::new(Url::parse(&endpoint)?))? - }, - Target::DB { path } => AptosDebugger::db(path)?, - }; - - println!( - "{:#?}", - debugger - .execute_past_transactions(args.begin_version, args.limit) - .await? - ); - - Ok(()) -} - -#[test] -fn verify_tool() { - use clap::CommandFactory; - Argument::command().debug_assert() -} diff --git a/crates/aptos-debugger/Cargo.toml b/crates/aptos-debugger/Cargo.toml new file mode 100644 index 0000000000000..35e8739d88f1b --- /dev/null +++ b/crates/aptos-debugger/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "aptos-debugger" +version = "0.1.0" +description = "Debugging tools." + +# Workspace inherited keys +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +aptos-db-tool = { workspace = true } +aptos-logger = { workspace = true } +aptos-move-debugger = { workspace = true } +aptos-push-metrics = { workspace = true } +clap = { workspace = true } +tokio = { workspace = true } diff --git a/crates/aptos-debugger/src/lib.rs b/crates/aptos-debugger/src/lib.rs new file mode 100644 index 0000000000000..498ad782eb127 --- /dev/null +++ b/crates/aptos-debugger/src/lib.rs @@ -0,0 +1,31 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use clap::Parser; + +#[derive(Parser)] +pub enum Cmd { + #[clap(subcommand)] + AptosDb(aptos_db_tool::DBTool), + + Decode(aptos_move_debugger::bcs_txn_decoder::Command), + + Move(aptos_move_debugger::aptos_debugger::Command), +} + +impl Cmd { + pub async fn run(self) -> Result<()> { + match self { + Cmd::AptosDb(cmd) => cmd.run().await, + Cmd::Decode(cmd) => cmd.run().await, + Cmd::Move(cmd) => cmd.run().await, + } + } +} + +#[test] +fn verify_tool() { + use clap::CommandFactory; + Cmd::command().debug_assert() +} diff --git a/storage/db-tool/src/main.rs b/crates/aptos-debugger/src/main.rs similarity index 81% rename from storage/db-tool/src/main.rs rename to crates/aptos-debugger/src/main.rs index f733536310e38..4df804804d703 100644 --- a/storage/db-tool/src/main.rs +++ b/crates/aptos-debugger/src/main.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; -use aptos_db_tool::DBTool; +use aptos_debugger::Cmd; use aptos_logger::{Level, Logger}; use aptos_push_metrics::MetricsPusher; use clap::Parser; @@ -12,6 +12,5 @@ async fn main() -> Result<()> { Logger::new().level(Level::Info).init(); let _mp = MetricsPusher::start(vec![]); - DBTool::parse().run().await?; - Ok(()) + Cmd::parse().run().await } diff --git a/crates/aptos/Cargo.toml b/crates/aptos/Cargo.toml index 55808aeb99f60..a9cb816204186 100644 --- a/crates/aptos/Cargo.toml +++ b/crates/aptos/Cargo.toml @@ -22,7 +22,6 @@ aptos-cached-packages = { workspace = true } aptos-cli-common = { workspace = true } aptos-config = { workspace = true } aptos-crypto = { workspace = true } -aptos-debugger = { workspace = true } aptos-faucet-core = { workspace = true } aptos-framework = { workspace = true } aptos-gas-profiling = { workspace = true } @@ -35,6 +34,7 @@ aptos-indexer-grpc-utils = { workspace = true } aptos-keygen = { workspace = true } aptos-ledger = { workspace = true } aptos-logger = { workspace = true } +aptos-move-debugger = { workspace = true } aptos-network-checker = { workspace = true } aptos-node = { workspace = true } aptos-protos = { workspace = true } diff --git a/crates/aptos/src/common/types.rs b/crates/aptos/src/common/types.rs index f77e5f9833bda..e143a26e9720c 100644 --- a/crates/aptos/src/common/types.rs +++ b/crates/aptos/src/common/types.rs @@ -23,11 +23,11 @@ use aptos_crypto::{ encoding_type::{EncodingError, EncodingType}, x25519, PrivateKey, ValidCryptoMaterialStringExt, }; -use aptos_debugger::AptosDebugger; use aptos_gas_profiling::FrameName; use aptos_global_constants::adjust_gas_headroom; use aptos_keygen::KeyGen; use aptos_logger::Level; +use aptos_move_debugger::aptos_debugger::AptosDebugger; use aptos_rest_client::{ aptos_api_types::{EntryFunctionId, HashValue, MoveType, ViewRequest}, error::RestError, diff --git a/docker/builder/README.md b/docker/builder/README.md index 0478748ce7767..d8aadea94bbe9 100644 --- a/docker/builder/README.md +++ b/docker/builder/README.md @@ -17,9 +17,9 @@ For using the images, look in the [docker/compose](../docker/compose/) directory The builder can produce the following Docker images. To build a particular image, run `./docker/builder/docker-bake-rust-all.sh [image-name]`. Also, refer to the `group` definitions in the [docker-bake-rust-all.hcl](docker-bake-rust-all.hcl) file for more information. -1. `validator-testing` : Image containing the `aptos-node` binary and other linux tools useful for debugging and testing. This image is used in Forge tests. -2. `validator` : Image containing only the `aptos-node` binary. This image is usually used for distribution. -3. `tools`: Image containing all the aptos tools binaries including `aptos-db-bootstrapper`, `aptos-db-tool`, `aptos`, `aptos-transaction-emitter`, `aptos-openapi-spec-generator` and `aptos-fn-check-client`. Also, includes the Aptos Move framework for use with genesis generation. +1. `validator-testing` : Image containing the `aptos-node`, `aptos-debugger` binaries and other linux tools useful for debugging and testing. This image is used in Forge tests. +2. `validator` : Image containing the `aptos-node` and `aptos-debugger` binaries. This image is usually used for distribution. +3. `tools`: Image containing all the aptos tools binaries including `aptos-debugger`, `aptos`, `aptos-transaction-emitter`, `aptos-openapi-spec-generator` and `aptos-fn-check-client`. Also, includes the Aptos Move framework for use with genesis generation. 4. `forge`: Image containing the `forge` binary that orchestrates and runs Forge tests. 5. `node-checker`: Image containing the `node-checker` binary that checks the health of a node. 6. `faucet`: Image containing the `faucet` binary that provides a faucet service for minting coins. diff --git a/docker/builder/build-tools.sh b/docker/builder/build-tools.sh index 54cc2e455e02a..945e1ba3da7fd 100755 --- a/docker/builder/build-tools.sh +++ b/docker/builder/build-tools.sh @@ -19,8 +19,7 @@ cargo build --locked --profile=$PROFILE \ -p aptos-node-checker \ -p aptos-openapi-spec-generator \ -p aptos-telemetry-service \ - -p aptos-db-bootstrapper \ - -p aptos-db-tool \ + -p aptos-debugger \ -p aptos-transaction-emitter \ -p aptos-indexer-grpc-cache-worker \ -p aptos-indexer-grpc-file-store \ @@ -38,8 +37,7 @@ BINS=( aptos-openapi-spec-generator aptos-telemetry-service aptos-fn-check-client - aptos-db-tool - aptos-db-bootstrapper + aptos-debugger forge aptos-transaction-emitter aptos-indexer-grpc-cache-worker diff --git a/docker/builder/tools.Dockerfile b/docker/builder/tools.Dockerfile index ab83c25e9c863..b275a602e5282 100644 --- a/docker/builder/tools.Dockerfile +++ b/docker/builder/tools.Dockerfile @@ -25,8 +25,7 @@ COPY --link docker/tools/boto.cfg /etc/boto.cfg RUN wget https://storage.googleapis.com/pub/gsutil.tar.gz -O- | tar --gzip --directory /opt --extract && ln -s /opt/gsutil/gsutil /usr/local/bin RUN cd /usr/local/bin && wget "https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/linux/amd64/kubectl" -O kubectl && chmod +x kubectl -COPY --link --from=tools-builder /aptos/dist/aptos-db-bootstrapper /usr/local/bin/aptos-db-bootstrapper -COPY --link --from=tools-builder /aptos/dist/aptos-db-tool /usr/local/bin/aptos-db-tool +COPY --link --from=tools-builder /aptos/dist/aptos-debugger /usr/local/bin/aptos-debugger COPY --link --from=tools-builder /aptos/dist/aptos /usr/local/bin/aptos COPY --link --from=tools-builder /aptos/dist/aptos-openapi-spec-generator /usr/local/bin/aptos-openapi-spec-generator COPY --link --from=tools-builder /aptos/dist/aptos-fn-check-client /usr/local/bin/aptos-fn-check-client diff --git a/docker/builder/validator-testing.Dockerfile b/docker/builder/validator-testing.Dockerfile index aadae1ffdb029..0171363e1d44c 100644 --- a/docker/builder/validator-testing.Dockerfile +++ b/docker/builder/validator-testing.Dockerfile @@ -43,8 +43,7 @@ RUN addgroup --system --gid 6180 aptos && adduser --system --ingroup aptos --no- RUN mkdir -p /opt/aptos/etc COPY --link --from=node-builder /aptos/dist/aptos-node /usr/local/bin/ -COPY --link --from=tools-builder /aptos/dist/aptos-db-tool /usr/local/bin/ -COPY --link --from=tools-builder /aptos/dist/aptos-db-bootstrapper /usr/local/bin/ +COPY --link --from=tools-builder /aptos/dist/aptos-debugger /usr/local/bin/ # Admission control EXPOSE 8000 @@ -67,4 +66,4 @@ ENV GIT_SHA ${GIT_SHA} # Capture backtrace on error ENV RUST_BACKTRACE 1 -ENV RUST_LOG_FORMAT=json \ No newline at end of file +ENV RUST_LOG_FORMAT=json diff --git a/docker/builder/validator.Dockerfile b/docker/builder/validator.Dockerfile index 7eb0ce807961e..db356cf63ed71 100644 --- a/docker/builder/validator.Dockerfile +++ b/docker/builder/validator.Dockerfile @@ -28,8 +28,7 @@ RUN addgroup --system --gid 6180 aptos && adduser --system --ingroup aptos --no- RUN mkdir -p /opt/aptos/etc COPY --link --from=node-builder /aptos/dist/aptos-node /usr/local/bin/ -COPY --link --from=tools-builder /aptos/dist/aptos-db-tool /usr/local/bin/ -COPY --link --from=tools-builder /aptos/dist/aptos-db-bootstrapper /usr/local/bin/ +COPY --link --from=tools-builder /aptos/dist/aptos-debugger /usr/local/bin/ # Admission control EXPOSE 8000 @@ -52,4 +51,4 @@ ENV GIT_TAG ${GIT_TAG} ARG GIT_BRANCH ENV GIT_BRANCH ${GIT_BRANCH} ARG GIT_SHA -ENV GIT_SHA ${GIT_SHA} \ No newline at end of file +ENV GIT_SHA ${GIT_SHA} diff --git a/docker/compose/data-restore/docker-compose.yaml b/docker/compose/data-restore/docker-compose.yaml index ebd72107cfcc2..792640fc6744f 100644 --- a/docker/compose/data-restore/docker-compose.yaml +++ b/docker/compose/data-restore/docker-compose.yaml @@ -25,7 +25,7 @@ services: - RUST_LOG=debug command: > sh -c " - /usr/local/bin/aptos-db-tool restore bootstrap-db --concurrent-downloads 2 \ + /usr/local/bin/aptos-debugger aptos-db restore bootstrap-db --concurrent-downloads 2 \ --target-db-dir /opt/aptos/data/db --metadata-cache-dir /tmp/aptos-restore-metadata \ --command-adapter-config /opt/aptos/etc/restore.yaml diff --git a/execution/db-bootstrapper/Cargo.toml b/execution/db-bootstrapper/Cargo.toml deleted file mode 100644 index a24894c8b01b2..0000000000000 --- a/execution/db-bootstrapper/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "aptos-db-bootstrapper" -description = "Aptos DB-Bootstrapper" -version = "0.1.0" - -# Workspace inherited keys -authors = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -publish = { workspace = true } -repository = { workspace = true } -rust-version = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -aptos-config = { workspace = true } -aptos-crypto = { workspace = true } -aptos-db = { workspace = true } -aptos-executor = { workspace = true } -aptos-storage-interface = { workspace = true } -aptos-temppath = { workspace = true } -aptos-types = { workspace = true } -aptos-vm = { workspace = true } -bcs = { workspace = true } -clap = { workspace = true } diff --git a/execution/db-bootstrapper/src/bin/aptos-db-bootstrapper.rs b/execution/db-bootstrapper/src/bin/aptos-db-bootstrapper.rs deleted file mode 100644 index a6b563930a35c..0000000000000 --- a/execution/db-bootstrapper/src/bin/aptos-db-bootstrapper.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright © Aptos Foundation -// Parts of the project are originally copyright © Meta Platforms, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::{ensure, format_err, Context, Result}; -use aptos_config::config::{ - RocksdbConfigs, StorageDirPaths, BUFFERED_STATE_TARGET_ITEMS, - DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG, -}; -use aptos_db::AptosDB; -use aptos_executor::db_bootstrapper::calculate_genesis; -use aptos_storage_interface::DbReaderWriter; -use aptos_types::{transaction::Transaction, waypoint::Waypoint}; -use aptos_vm::AptosVM; -use clap::Parser; -use std::{ - fs::File, - io::Read, - path::{Path, PathBuf}, -}; - -#[derive(Parser)] -#[clap( - name = "aptos-db-bootstrapper", - about = "Calculate, verify and commit the genesis to local DB without a consensus among validators." -)] -struct Opt { - #[clap(value_parser)] - db_dir: PathBuf, - - #[clap(short, long, value_parser)] - genesis_txn_file: PathBuf, - - #[clap(short, long)] - waypoint_to_verify: Option, - - #[clap(long, requires("waypoint_to_verify"))] - commit: bool, -} - -fn main() -> Result<()> { - let opt = Opt::parse(); - - let genesis_txn = load_genesis_txn(&opt.genesis_txn_file) - .with_context(|| format_err!("Failed loading genesis txn."))?; - assert!( - matches!(genesis_txn, Transaction::GenesisTransaction(_)), - "Not a GenesisTransaction" - ); - - // Opening the DB exclusively, it's not allowed to run this tool alongside a running node which - // operates on the same DB. - let db = AptosDB::open( - StorageDirPaths::from_path(&opt.db_dir), - false, - NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */ - RocksdbConfigs::default(), - false, /* indexer */ - BUFFERED_STATE_TARGET_ITEMS, - DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, - ) - .expect("Failed to open DB."); - let db = DbReaderWriter::new(db); - - let executed_trees = db - .reader - .get_latest_executed_trees() - .with_context(|| format_err!("Failed to get latest tree state."))?; - println!("Db has {} transactions", executed_trees.num_transactions()); - if let Some(waypoint) = opt.waypoint_to_verify { - ensure!( - waypoint.version() == executed_trees.num_transactions(), - "Trying to generate waypoint at version {}, but DB has {} transactions.", - waypoint.version(), - executed_trees.num_transactions(), - ) - } - - let committer = calculate_genesis::(&db, executed_trees, &genesis_txn) - .with_context(|| format_err!("Failed to calculate genesis."))?; - println!( - "Successfully calculated genesis. Got waypoint: {}", - committer.waypoint() - ); - - if let Some(waypoint) = opt.waypoint_to_verify { - ensure!( - waypoint == committer.waypoint(), - "Waypoint verification failed. Expected {:?}, got {:?}.", - waypoint, - committer.waypoint(), - ); - println!("Waypoint verified."); - - if opt.commit { - committer - .commit() - .with_context(|| format_err!("Committing genesis to DB."))?; - println!("Successfully committed genesis.") - } - } - - Ok(()) -} - -fn load_genesis_txn(path: &Path) -> Result { - let mut file = File::open(path)?; - let mut buffer = vec![]; - file.read_to_end(&mut buffer)?; - - Ok(bcs::from_bytes(&buffer)?) -} - -#[test] -fn verify_tool() { - use clap::CommandFactory; - Opt::command().debug_assert() -} diff --git a/storage/README.md b/storage/README.md index f7b8abb8bac8c..e22c36bb7dcf0 100644 --- a/storage/README.md +++ b/storage/README.md @@ -172,14 +172,14 @@ https://github.com/aptos-labs/aptos-core/tree/main/storage/backup/backup-cli/src ```bash -$ cargo run -p aptos-db-tool backup continuously --help +$ cargo run -p aptos-debugger aptos-db backup continuously --help Finished dev [unoptimized + debuginfo] target(s) in 1.06s - Running `target/debug/aptos-db-tool backup continuously --help` + Running `target/debug/aptos-debugger aptos-db backup continuously --help` aptos-db-tool-backup-continuously 0.1.0 Run the backup coordinator which backs up blockchain data continuously off a Aptos Node. USAGE: - aptos-db-tool backup continuously [OPTIONS] <--local-fs-dir |--command-adapter-config > + aptos-debugger aptos-db backup continuously [OPTIONS] <--local-fs-dir |--command-adapter-config > OPTIONS: --backup-service-address
@@ -237,14 +237,14 @@ OPTIONS: Example command: ``` -$ cargo run -p aptos-db-tool backup continuously \ +$ cargo run -p aptos-debugger aptos-db backup continuously \ --metadata-cache-dir ./mc \ --state-snapshot-interval-epochs 1 \ --concurrent-downloads 4 \ --command-adapter-config s3.yaml ``` -There are other subcommands of the aptos-db-tool, all of which are experimental +There are other subcommands of the aptos-debugger aptos-db, all of which are experimental and can mess up with the backup storage, use only at your own risk. ### Creating an AptosDB with minimal data at the latest epoch ending in a backup @@ -304,7 +304,7 @@ RUST_LOG=info ./aptos \ ``` This is basically the same functionality with -the "auto" mode of `cargo run -p aptos-db-tool restore`, but with more +the "auto" mode of `cargo run -p aptos-debugger aptos-db restore`, but with more limited options. The `restore` tool mentioned has the ability to manually hack a local DB and is highly experimental. It's not recommended is be used if you are not 100% aware of what you are doing. diff --git a/storage/aptosdb/Cargo.toml b/storage/aptosdb/Cargo.toml index b3d14f6cc63ab..ec1b2321376d4 100644 --- a/storage/aptosdb/Cargo.toml +++ b/storage/aptosdb/Cargo.toml @@ -74,8 +74,3 @@ default = [] fuzzing = ["proptest", "proptest-derive", "aptos-proptest-helpers", "aptos-temppath", "aptos-crypto/fuzzing", "aptos-jellyfish-merkle/fuzzing", "aptos-types/fuzzing", "aptos-executor-types/fuzzing", "aptos-schemadb/fuzzing", "aptos-scratchpad/fuzzing"] consensus-only-perf-test = [] db-debugger = ["aptos-temppath", "clap", "owo-colors"] - -[[bin]] -name = "db-debugger" -path = "src/db_debugger/main.rs" -required-features = ["db-debugger"] diff --git a/storage/aptosdb/src/db_debugger/main.rs b/storage/aptosdb/src/db_debugger/main.rs deleted file mode 100644 index 8d7d5c35ccc33..0000000000000 --- a/storage/aptosdb/src/db_debugger/main.rs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::Result; -use aptos_db::db_debugger::Cmd; -use clap::Parser; - -fn main() -> Result<()> { - Cmd::parse().run() -} diff --git a/storage/db-tool/Cargo.toml b/storage/db-tool/Cargo.toml index 5eddfb8acf58a..e6fbc302abead 100644 --- a/storage/db-tool/Cargo.toml +++ b/storage/db-tool/Cargo.toml @@ -17,6 +17,7 @@ aptos-backup-cli = { workspace = true } aptos-backup-service = { workspace = true } aptos-config = { workspace = true } aptos-db = { workspace = true, features = ["db-debugger"] } +aptos-executor = { workspace = true } aptos-executor-types = { workspace = true } aptos-logger = { workspace = true } aptos-push-metrics = { workspace = true } @@ -24,7 +25,9 @@ aptos-state-view = { workspace = true } aptos-storage-interface = { workspace = true } aptos-temppath = { workspace = true } aptos-types = { workspace = true } +aptos-vm = { workspace = true } async-trait = { workspace = true } +bcs = { workspace = true } clap = { workspace = true } itertools = { workspace = true } owo-colors = { workspace = true } diff --git a/storage/db-tool/src/bootstrap.rs b/storage/db-tool/src/bootstrap.rs new file mode 100644 index 0000000000000..46aed10777957 --- /dev/null +++ b/storage/db-tool/src/bootstrap.rs @@ -0,0 +1,118 @@ +// Copyright © Aptos Foundation +// Parts of the project are originally copyright © Meta Platforms, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{ensure, format_err, Context, Result}; +use aptos_config::config::{ + RocksdbConfigs, StorageDirPaths, BUFFERED_STATE_TARGET_ITEMS, + DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG, +}; +use aptos_db::AptosDB; +use aptos_executor::db_bootstrapper::calculate_genesis; +use aptos_storage_interface::DbReaderWriter; +use aptos_types::{transaction::Transaction, waypoint::Waypoint}; +use aptos_vm::AptosVM; +use clap::Parser; +use std::{ + fs::File, + io::Read, + path::{Path, PathBuf}, +}; + +#[derive(Parser)] +#[clap( + name = "aptos-db-bootstrapper", + about = "Calculate, verify and commit the genesis to local DB without a consensus among validators." +)] +pub struct Command { + #[clap(value_parser)] + db_dir: PathBuf, + + #[clap(short, long, value_parser)] + genesis_txn_file: PathBuf, + + #[clap(short, long)] + waypoint_to_verify: Option, + + #[clap(long, requires("waypoint_to_verify"))] + commit: bool, +} + +impl Command { + pub fn run(self) -> Result<()> { + let genesis_txn = load_genesis_txn(&self.genesis_txn_file) + .with_context(|| format_err!("Failed loading genesis txn."))?; + assert!( + matches!(genesis_txn, Transaction::GenesisTransaction(_)), + "Not a GenesisTransaction" + ); + + // Opening the DB exclusively, it's not allowed to run this tool alongside a running node which + // operates on the same DB. + let db = AptosDB::open( + StorageDirPaths::from_path(&self.db_dir), + false, + NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */ + RocksdbConfigs::default(), + false, /* indexer */ + BUFFERED_STATE_TARGET_ITEMS, + DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, + ) + .expect("Failed to open DB."); + let db = DbReaderWriter::new(db); + + let executed_trees = db + .reader + .get_latest_executed_trees() + .with_context(|| format_err!("Failed to get latest tree state."))?; + println!("Db has {} transactions", executed_trees.num_transactions()); + if let Some(waypoint) = self.waypoint_to_verify { + ensure!( + waypoint.version() == executed_trees.num_transactions(), + "Trying to generate waypoint at version {}, but DB has {} transactions.", + waypoint.version(), + executed_trees.num_transactions(), + ) + } + + let committer = calculate_genesis::(&db, executed_trees, &genesis_txn) + .with_context(|| format_err!("Failed to calculate genesis."))?; + println!( + "Successfully calculated genesis. Got waypoint: {}", + committer.waypoint() + ); + + if let Some(waypoint) = self.waypoint_to_verify { + ensure!( + waypoint == committer.waypoint(), + "Waypoint verification failed. Expected {:?}, got {:?}.", + waypoint, + committer.waypoint(), + ); + println!("Waypoint verified."); + + if self.commit { + committer + .commit() + .with_context(|| format_err!("Committing genesis to DB."))?; + println!("Successfully committed genesis.") + } + } + + Ok(()) + } +} + +fn load_genesis_txn(path: &Path) -> Result { + let mut file = File::open(path)?; + let mut buffer = vec![]; + file.read_to_end(&mut buffer)?; + + Ok(bcs::from_bytes(&buffer)?) +} + +#[test] +fn verify_tool() { + use clap::CommandFactory; + Command::command().debug_assert() +} diff --git a/storage/db-tool/src/debugger.rs b/storage/db-tool/src/debugger.rs deleted file mode 100644 index 6ed638841ed4a..0000000000000 --- a/storage/db-tool/src/debugger.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::Result; -use aptos_db::db_debugger::{checkpoint, ledger, state_tree, truncate}; -use clap::Parser; - -/// List snapshots, print nodes, make DB checkpoints and validate ledger hash -#[derive(Parser)] -pub enum Command { - #[clap(subcommand)] - StateTree(state_tree::Cmd), - Checkpoint(checkpoint::Cmd), - #[clap(subcommand)] - Ledger(ledger::Cmd), - Truncate(truncate::Cmd), -} - -impl Command { - pub fn run(self) -> Result<()> { - match self { - Command::StateTree(cmd) => cmd.run(), - Command::Checkpoint(cmd) => cmd.run(), - Command::Ledger(cmd) => cmd.run(), - Command::Truncate(cmd) => cmd.run(), - } - } -} diff --git a/storage/db-tool/src/lib.rs b/storage/db-tool/src/lib.rs index 5615992335530..3b22be029f847 100644 --- a/storage/db-tool/src/lib.rs +++ b/storage/db-tool/src/lib.rs @@ -5,7 +5,7 @@ extern crate core; mod backup; mod backup_maintenance; -mod debugger; +mod bootstrap; mod replay_verify; pub mod restore; #[cfg(test)] @@ -13,6 +13,7 @@ mod tests; mod utils; use anyhow::Result; +use aptos_db::db_debugger; use clap::Parser; #[derive(Parser)] @@ -20,23 +21,30 @@ use clap::Parser; pub enum DBTool { #[clap(subcommand)] Backup(backup::Command), + #[clap(subcommand)] - Restore(restore::Command), - ReplayVerify(replay_verify::Opt), + BackupMaintenance(backup_maintenance::Command), + + Bootstrap(bootstrap::Command), + #[clap(subcommand)] - Debug(debugger::Command), + Debug(db_debugger::Cmd), + + ReplayVerify(replay_verify::Opt), + #[clap(subcommand)] - BackupMaintenance(backup_maintenance::Command), + Restore(restore::Command), } impl DBTool { pub async fn run(self) -> Result<()> { match self { DBTool::Backup(cmd) => cmd.run().await, - DBTool::Restore(cmd) => cmd.run().await, - DBTool::ReplayVerify(cmd) => cmd.run().await, DBTool::BackupMaintenance(cmd) => cmd.run().await, + DBTool::Bootstrap(cmd) => cmd.run(), DBTool::Debug(cmd) => cmd.run(), + DBTool::ReplayVerify(cmd) => cmd.run().await, + DBTool::Restore(cmd) => cmd.run().await, } } } diff --git a/terraform/helm/fullnode/templates/backup-compaction.yaml b/terraform/helm/fullnode/templates/backup-compaction.yaml index a71dc360d9564..f09083d3947a2 100644 --- a/terraform/helm/fullnode/templates/backup-compaction.yaml +++ b/terraform/helm/fullnode/templates/backup-compaction.yaml @@ -31,7 +31,8 @@ spec: {{- end }} imagePullPolicy: {{ .Values.backup.image.pullPolicy }} command: - - /usr/local/bin/aptos-db-tool + - /usr/local/bin/aptos-debugger + - aptos-db - backup-maintenance - compact - --state-snapshot-file-compact-factor @@ -101,4 +102,4 @@ spec: {{- if .Values.imagePullSecret }} imagePullSecrets: - name: {{.Values.imagePullSecret}} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/terraform/helm/fullnode/templates/backup-verify.yaml b/terraform/helm/fullnode/templates/backup-verify.yaml index d5900203888d8..08c3ae779299a 100644 --- a/terraform/helm/fullnode/templates/backup-verify.yaml +++ b/terraform/helm/fullnode/templates/backup-verify.yaml @@ -31,7 +31,8 @@ spec: {{- end }} imagePullPolicy: {{ .Values.backup.image.pullPolicy }} command: - - /usr/local/bin/aptos-db-tool + - /usr/local/bin/aptos-debugger + - aptos-db - backup - verify {{- range $.Values.restore.config.trusted_waypoints }} diff --git a/terraform/helm/fullnode/templates/backup.yaml b/terraform/helm/fullnode/templates/backup.yaml index 35946bd323ff8..71182ba4cd064 100644 --- a/terraform/helm/fullnode/templates/backup.yaml +++ b/terraform/helm/fullnode/templates/backup.yaml @@ -43,8 +43,9 @@ spec: imagePullPolicy: {{ .Values.backup.image.pullPolicy }} resources: {{- toYaml .Values.backup.resources | nindent 10 }} - command: ["/usr/local/bin/aptos-db-tool"] + command: ["/usr/local/bin/aptos-debugger"] args: + - "aptos-db" - "backup" - "continuously" - "--metadata-cache-dir" diff --git a/terraform/helm/fullnode/templates/restore.yaml b/terraform/helm/fullnode/templates/restore.yaml index a53ce9f5190a4..575167ac3530c 100644 --- a/terraform/helm/fullnode/templates/restore.yaml +++ b/terraform/helm/fullnode/templates/restore.yaml @@ -41,7 +41,7 @@ spec: echo "$CONTROLLER_UID" > /opt/aptos/data/restore-uid fi # start restore process - /usr/local/bin/aptos-db-tool restore bootstrap-db --concurrent-downloads {{ .config.concurrent_downloads }}{{ range .config.trusted_waypoints }} --trust-waypoint {{ . }}{{ end }} --target-db-dir /opt/aptos/data/db --metadata-cache-dir /tmp/aptos-restore-metadata --command-adapter-config /opt/aptos/etc/{{ .config.location }}.yaml + /usr/local/bin/aptos-debugger aptos-db restore bootstrap-db --concurrent-downloads {{ .config.concurrent_downloads }}{{ range .config.trusted_waypoints }} --trust-waypoint {{ . }}{{ end }} --target-db-dir /opt/aptos/data/db --metadata-cache-dir /tmp/aptos-restore-metadata --command-adapter-config /opt/aptos/etc/{{ .config.location }}.yaml env: - name: RUST_LOG value: "debug" diff --git a/testsuite/module_verify.py b/testsuite/module_verify.py index 02dd1bb07d0e7..8adcc55590c50 100755 --- a/testsuite/module_verify.py +++ b/testsuite/module_verify.py @@ -39,7 +39,8 @@ def main(): os.mkdir("local") subprocess.run( [ - "target/release/aptos-db-tool", + "target/release/aptos-debugger", + "aptos-db", "backup", "verify", "--validate-modules", diff --git a/testsuite/module_verify_run_local.py b/testsuite/module_verify_run_local.py index 3d534b4fa97b5..58d39e918da32 100755 --- a/testsuite/module_verify_run_local.py +++ b/testsuite/module_verify_run_local.py @@ -29,7 +29,7 @@ def local_setup(): "build", "--release", "-p", - "aptos-db-tool", + "aptos-debugger", ], check=True, ) diff --git a/testsuite/replay_verify.py b/testsuite/replay_verify.py index f7aa999df9f6b..9718210ff2458 100755 --- a/testsuite/replay_verify.py +++ b/testsuite/replay_verify.py @@ -15,7 +15,7 @@ from verify_core.common import clear_artifacts, query_backup_latest_version # This script runs the replay-verify from the root of aptos-core -# It assumes the aptos-db-tool binary is already built with the release profile +# It assumes the aptos-debugger binary is already built with the release profile # The key is the runner's number and the value is the range of txns that the runner is responsible for @@ -109,7 +109,8 @@ def replay_verify_partition( # run and print output process = subprocess.Popen( [ - "target/release/aptos-db-tool", + "target/release/aptos-debugger", + "aptos-db", "replay-verify", *txns_to_skip_args, "--concurrent-downloads", diff --git a/testsuite/replay_verify_run_local.py b/testsuite/replay_verify_run_local.py index 64aeee413b66c..0beff5adc39eb 100755 --- a/testsuite/replay_verify_run_local.py +++ b/testsuite/replay_verify_run_local.py @@ -33,7 +33,7 @@ def local_setup(): "build", "--release", "-p", - "aptos-db-tool", + "aptos-debugger", ], check=True, ) diff --git a/testsuite/smoke-test/Cargo.toml b/testsuite/smoke-test/Cargo.toml index 482079d28dc70..5819102b0c470 100644 --- a/testsuite/smoke-test/Cargo.toml +++ b/testsuite/smoke-test/Cargo.toml @@ -20,7 +20,6 @@ aptos-cached-packages = { workspace = true } aptos-config = { workspace = true } aptos-consensus = { workspace = true } aptos-crypto = { workspace = true } -aptos-debugger = { workspace = true } aptos-faucet-core = { workspace = true } aptos-forge = { workspace = true } aptos-framework = { workspace = true } @@ -30,6 +29,7 @@ aptos-global-constants = { workspace = true } aptos-indexer = { workspace = true } aptos-inspection-service = { workspace = true } aptos-keygen = { workspace = true } +aptos-move-debugger = { workspace = true } aptos-release-builder = { workspace = true } aptos-rest-client = { workspace = true } aptos-rosetta = { workspace = true } diff --git a/testsuite/smoke-test/src/aptos/mint_transfer.rs b/testsuite/smoke-test/src/aptos/mint_transfer.rs index 8d3739d3400cb..dbc194f1da6a1 100644 --- a/testsuite/smoke-test/src/aptos/mint_transfer.rs +++ b/testsuite/smoke-test/src/aptos/mint_transfer.rs @@ -3,8 +3,8 @@ use crate::smoke_test_environment::new_local_swarm_with_aptos; use aptos_cached_packages::aptos_stdlib; -use aptos_debugger::AptosDebugger; use aptos_forge::Swarm; +use aptos_move_debugger::aptos_debugger::AptosDebugger; use aptos_types::transaction::{ExecutionStatus, TransactionStatus}; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] diff --git a/testsuite/smoke-test/src/genesis.rs b/testsuite/smoke-test/src/genesis.rs index 526b7d3d27b01..0c8b0e51eb265 100644 --- a/testsuite/smoke-test/src/genesis.rs +++ b/testsuite/smoke-test/src/genesis.rs @@ -68,12 +68,9 @@ async fn wait_for_node(validator: &mut dyn Validator, expected_to_connect: usize /// 3. Test the nodes and clients resume working after updating waypoint /// 4. Test a node lagging behind can sync to the waypoint async fn test_genesis_transaction_flow() { - let db_bootstrapper = workspace_builder::get_bin("aptos-db-bootstrapper"); + let aptos_debugger = workspace_builder::get_bin("aptos-debugger"); let aptos_cli = workspace_builder::get_bin("aptos"); - // prebuild tools. - workspace_builder::get_bin("aptos-db-tool"); - println!("0. pre-building finished."); let num_nodes = 5; @@ -182,9 +179,11 @@ async fn test_genesis_transaction_flow() { }; println!("6. prepare the waypoint with the transaction"); - let waypoint_command = Command::new(db_bootstrapper.as_path()) + let waypoint_command = Command::new(aptos_debugger.as_path()) .current_dir(workspace_root()) .args(&vec![ + "aptos-db", + "bootstrap", env.validators() .next() .unwrap() diff --git a/testsuite/smoke-test/src/storage.rs b/testsuite/smoke-test/src/storage.rs index 268e91fd004ce..d60437909cc76 100644 --- a/testsuite/smoke-test/src/storage.rs +++ b/testsuite/smoke-test/src/storage.rs @@ -29,7 +29,7 @@ async fn test_db_restore() { // pre-build tools ::aptos_logger::Logger::new().init(); info!("---------- 0. test_db_restore started."); - workspace_builder::get_bin("aptos-db-tool"); + workspace_builder::get_bin("aptos-debugger"); info!("---------- 1. pre-building finished."); let mut swarm = SwarmBuilder::new_local(4).with_aptos().build().await; @@ -179,15 +179,15 @@ async fn test_db_restore() { } fn db_backup_verify(backup_path: &Path, trusted_waypoints: &[Waypoint]) { - info!("---------- running aptos-db-tool backup-verify"); + info!("---------- running aptos-debugger aptos-db backup-verify"); let now = Instant::now(); - let bin_path = workspace_builder::get_bin("aptos-db-tool"); + let bin_path = workspace_builder::get_bin("aptos-debugger"); let metadata_cache_path = TempPath::new(); metadata_cache_path.create_as_dir().unwrap(); let mut cmd = Command::new(bin_path.as_path()); - cmd.args(["backup", "verify"]); + cmd.args(["aptos-db", "backup", "verify"]); trusted_waypoints.iter().for_each(|w| { cmd.arg("--trust-waypoint"); cmd.arg(&w.to_string()); @@ -212,14 +212,14 @@ fn db_backup_verify(backup_path: &Path, trusted_waypoints: &[Waypoint]) { fn replay_verify(backup_path: &Path, trusted_waypoints: &[Waypoint]) { info!("---------- running replay-verify"); let now = Instant::now(); - let bin_path = workspace_builder::get_bin("aptos-db-tool"); + let bin_path = workspace_builder::get_bin("aptos-debugger"); let metadata_cache_path = TempPath::new(); let target_db_dir = TempPath::new(); metadata_cache_path.create_as_dir().unwrap(); let mut cmd = Command::new(bin_path.as_path()); - cmd.arg("replay-verify"); + cmd.args(["aptos-db", "replay-verify"]); trusted_waypoints.iter().for_each(|w| { cmd.arg("--trust-waypoint"); cmd.arg(&w.to_string()); @@ -302,6 +302,7 @@ fn get_backup_storage_state( let output = Command::new(bin_path) .current_dir(workspace_root()) .args([ + "aptos-db", "backup", "query", "backup-storage-state", @@ -327,7 +328,7 @@ pub(crate) fn db_backup( ) -> (TempPath, Version) { info!("---------- running aptos db tool backup"); let now = Instant::now(); - let bin_path = workspace_builder::get_bin("aptos-db-tool"); + let bin_path = workspace_builder::get_bin("aptos-debugger"); let metadata_cache_path1 = TempPath::new(); let metadata_cache_path2 = TempPath::new(); let backup_path = TempPath::new(); @@ -344,6 +345,7 @@ pub(crate) fn db_backup( let mut backup_coordinator = Command::new(bin_path.as_path()) .current_dir(workspace_root()) .args([ + "aptos-db", "backup", "continuously", "--backup-service-address", @@ -377,6 +379,7 @@ pub(crate) fn db_backup( let compaction = Command::new(bin_path.as_path()) .current_dir(workspace_root()) .args([ + "aptos-db", "backup-maintenance", "compact", "--epoch-ending-file-compact-factor", @@ -413,13 +416,13 @@ pub(crate) fn db_restore( target_verion: Option, /* target version should be same as epoch ending version to start a node */ ) { let now = Instant::now(); - let bin_path = workspace_builder::get_bin("aptos-db-tool"); + let bin_path = workspace_builder::get_bin("aptos-debugger"); let metadata_cache_path = TempPath::new(); metadata_cache_path.create_as_dir().unwrap(); let mut cmd = Command::new(bin_path.as_path()); - cmd.args(["restore", "bootstrap-db"]); + cmd.args(["aptos-db", "restore", "bootstrap-db"]); trusted_waypoints.iter().for_each(|w| { cmd.arg("--trust-waypoint"); cmd.arg(&w.to_string()); diff --git a/testsuite/verify_core/common.py b/testsuite/verify_core/common.py index 8996b99de8620..a7ae25d42fccc 100644 --- a/testsuite/verify_core/common.py +++ b/testsuite/verify_core/common.py @@ -43,7 +43,8 @@ def query_backup_latest_version(backup_config_template_path: str) -> int: """query latest version in backup, at the same time, pre-heat metadata cache""" db_backup_result = subprocess.Popen( [ - "target/release/aptos-db-tool", + "target/release/aptos-debugger", + "aptos-db", "backup", "query", "backup-storage-state",