[api] Add consistent API testing

.dockerignore

@@ -22,6 +22,7 @@
 !aptos-move/aptos-release-builder/data/release.yaml
 !aptos-move/aptos-release-builder/data/proposals/*
 !aptos-move/framework/
+!aptos-move/move-examples/hello_blockchain/
 !crates/aptos/src/move_tool/*.bpl
 !crates/aptos-faucet/doc/
 !api/doc/

Cargo.toml

@@ -48,6 +48,7 @@ members = [
     "consensus/consensus-types",
     "consensus/safety-rules",
     "crates/aptos",
+    "crates/aptos-api-tester",
     "crates/aptos-bitvec",
     "crates/aptos-build-info",
     "crates/aptos-compression",

crates/aptos-api-tester/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "aptos-api-tester"
+description = "Aptos developer API tester"
+version = "0.1.0"
+
+# Workspace inherited keys
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+
+[dependencies]
+anyhow = { workspace = true }
+aptos-api-types = { workspace = true }
+aptos-cached-packages = { workspace = true }
+aptos-framework = { workspace = true }
+aptos-logger = { workspace = true }
+aptos-network = { workspace = true }
+aptos-push-metrics = { workspace = true }
+aptos-rest-client = { workspace = true }
+aptos-sdk = { workspace = true }
+aptos-types = { workspace = true }
+futures = { workspace = true }
+move-core-types = { workspace = true }
+once_cell = { workspace = true }
+prometheus = { workspace = true }
+rand = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true }
+url = { workspace = true }

crates/aptos-api-tester/src/consts.rs

@@ -0,0 +1,68 @@
+// Copyright © Aptos Foundation
+
+use crate::utils::NetworkName;
+use once_cell::sync::Lazy;
+use std::{env, time::Duration};
+use url::Url;
+
+// Node and faucet constants
+
+// TODO: consider making this a CLI argument
+pub static NETWORK_NAME: Lazy<NetworkName> = Lazy::new(|| {
+    env::var("NETWORK_NAME")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(NetworkName::Devnet)
+});
+
+pub static DEVNET_NODE_URL: Lazy<Url> =
+    Lazy::new(|| Url::parse("https://fullnode.devnet.aptoslabs.com").unwrap());
+
+pub static DEVNET_FAUCET_URL: Lazy<Url> =
+    Lazy::new(|| Url::parse("https://faucet.devnet.aptoslabs.com").unwrap());
+
+pub static TESTNET_NODE_URL: Lazy<Url> =
+    Lazy::new(|| Url::parse("https://fullnode.testnet.aptoslabs.com").unwrap());
+
+pub static TESTNET_FAUCET_URL: Lazy<Url> =
+    Lazy::new(|| Url::parse("https://faucet.testnet.aptoslabs.com").unwrap());
+
+pub const FUND_AMOUNT: u64 = 100_000_000;
+
+// Persistency check constants
+
+// How long a persistent check runs for.
+pub static PERSISTENCY_TIMEOUT: Lazy<Duration> = Lazy::new(|| {
+    env::var("PERSISTENCY_TIMEOUT")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .map(Duration::from_secs)
+        .unwrap_or(Duration::from_secs(30))
+});
+
+// Wait time between tries during a persistent check.
+pub static SLEEP_PER_CYCLE: Lazy<Duration> = Lazy::new(|| {
+    env::var("SLEEP_PER_CYCLE")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .map(Duration::from_millis)
+        .unwrap_or(Duration::from_millis(100))
+});
+
+// Runtime constants
+
+// The number of threads to use for running tests.
+pub static NUM_THREADS: Lazy<usize> = Lazy::new(|| {
+    env::var("NUM_THREADS")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(4)
+});
+
+// The size of the stack for each thread.
+pub static STACK_SIZE: Lazy<usize> = Lazy::new(|| {
+    env::var("STACK_SIZE")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(4 * 1024 * 1024)
+});

crates/aptos-api-tester/src/counters.rs

@@ -0,0 +1,75 @@
+// Copyright © Aptos Foundation
+
+use once_cell::sync::Lazy;
+use prometheus::{register_histogram_vec, Histogram, HistogramVec};
+
+pub static API_TEST_SUCCESS: Lazy<HistogramVec> = Lazy::new(|| {
+    register_histogram_vec!(
+        "api_test_success",
+        "Number of user flows which succesfully passed",
+        &["test_name", "network_name", "run_id"],
+    )
+    .unwrap()
+});
+
+pub fn test_success(test_name: &str, network_name: &str, run_id: &str) -> Histogram {
+    API_TEST_SUCCESS.with_label_values(&[test_name, network_name, run_id])
+}
+
+pub static API_TEST_FAIL: Lazy<HistogramVec> = Lazy::new(|| {
+    register_histogram_vec!(
+        "api_test_fail",
+        "Number of user flows which failed checks",
+        &["test_name", "network_name", "run_id"],
+    )
+    .unwrap()
+});
+
+pub fn test_fail(test_name: &str, network_name: &str, run_id: &str) -> Histogram {
+    API_TEST_FAIL.with_label_values(&[test_name, network_name, run_id])
+}
+
+pub static API_TEST_ERROR: Lazy<HistogramVec> = Lazy::new(|| {
+    register_histogram_vec!("api_test_error", "Number of user flows which crashed", &[
+        "test_name",
+        "network_name",
+        "run_id"
+    ],)
+    .unwrap()
+});
+
+pub fn test_error(test_name: &str, network_name: &str, run_id: &str) -> Histogram {
+    API_TEST_ERROR.with_label_values(&[test_name, network_name, run_id])
+}
+
+pub static API_TEST_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
+    register_histogram_vec!(
+        "api_test_latency",
+        "Time it takes to complete a user flow",
+        &["test_name", "network_name", "run_id", "result"],
+    )
+    .unwrap()
+});
+
+pub fn test_latency(test_name: &str, network_name: &str, run_id: &str, result: &str) -> Histogram {
+    API_TEST_LATENCY.with_label_values(&[test_name, network_name, run_id, result])
+}
+
+pub static API_TEST_STEP_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
+    register_histogram_vec!(
+        "api_test_step_latency",
+        "Time it takes to complete a user flow step",
+        &["test_name", "step_name", "network_name", "run_id", "result"],
+    )
+    .unwrap()
+});
+
+pub fn test_step_latency(
+    test_name: &str,
+    step_name: &str,
+    network_name: &str,
+    run_id: &str,
+    result: &str,
+) -> Histogram {
+    API_TEST_STEP_LATENCY.with_label_values(&[test_name, step_name, network_name, run_id, result])
+}

crates/aptos-api-tester/src/macros.rs

@@ -0,0 +1,18 @@
+// Copyright © Aptos Foundation
+
+#[macro_export]
+macro_rules! time_fn {
+    ($func:expr, $($arg:expr), *) => {{
+        // start timer
+        let start = tokio::time::Instant::now();
+
+        // call the flow
+        let result = $func($($arg),+).await;
+
+        // end timer
+        let time = (tokio::time::Instant::now() - start).as_micros() as f64;
+
+        // return
+        (result, time)
+    }};
+}

crates/aptos-api-tester/src/main.rs

@@ -0,0 +1,97 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+#![forbid(unsafe_code)]
+
+mod consts;
+mod counters;
+mod persistent_check;
+mod strings;
+mod tests;
+mod tokenv1_client;
+mod utils;
+#[macro_use]
+mod macros;
+
+use crate::utils::{NetworkName, TestName};
+use anyhow::Result;
+use aptos_logger::{info, Level, Logger};
+use aptos_push_metrics::MetricsPusher;
+use consts::{NETWORK_NAME, NUM_THREADS, STACK_SIZE};
+use futures::future::join_all;
+use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::runtime::{Builder, Runtime};
+
+async fn test_flows(runtime: &Runtime, network_name: NetworkName) -> Result<()> {
+    let run_id = SystemTime::now()
+        .duration_since(UNIX_EPOCH)?
+        .as_secs()
+        .to_string();
+    info!(
+        "----- STARTING TESTS FOR {} WITH RUN ID {} -----",
+        network_name.to_string(),
+        run_id
+    );
+
+    // Flow 1: New account
+    let test_time = run_id.clone();
+    let handle_newaccount = runtime.spawn(async move {
+        TestName::NewAccount.run(network_name, &test_time).await;
+    });
+
+    // Flow 2: Coin transfer
+    let test_time = run_id.clone();
+    let handle_cointransfer = runtime.spawn(async move {
+        TestName::CoinTransfer.run(network_name, &test_time).await;
+    });
+
+    // Flow 3: NFT transfer
+    let test_time = run_id.clone();
+    let handle_nfttransfer = runtime.spawn(async move {
+        TestName::TokenV1Transfer
+            .run(network_name, &test_time)
+            .await;
+    });
+
+    // Flow 4: Publishing module
+    let test_time = run_id.clone();
+    let handle_publishmodule = runtime.spawn(async move {
+        TestName::PublishModule.run(network_name, &test_time).await;
+    });
+
+    // Flow 5: View function
+    let test_time = run_id.clone();
+    let handle_viewfunction = runtime.spawn(async move {
+        TestName::ViewFunction.run(network_name, &test_time).await;
+    });
+
+    join_all(vec![
+        handle_newaccount,
+        handle_cointransfer,
+        handle_nfttransfer,
+        handle_publishmodule,
+        handle_viewfunction,
+    ])
+    .await;
+    Ok(())
+}
+
+fn main() -> Result<()> {
+    // create runtime
+    let runtime = Builder::new_multi_thread()
+        .worker_threads(*NUM_THREADS)
+        .enable_all()
+        .thread_stack_size(*STACK_SIZE)
+        .build()?;
+
+    // log metrics
+    Logger::builder().level(Level::Info).build();
+    let _mp = MetricsPusher::start_for_local_run("api-tester");
+
+    // run tests
+    runtime.block_on(async {
+        let _ = test_flows(&runtime, *NETWORK_NAME).await;
+    });
+
+    Ok(())
+}