Merge pull request #3355 from stacks-network/release/2.05.0.5.0

2.05.0.5.0 RC1

.github/workflows/bitcoin-tests.yml

@@ -40,6 +40,7 @@ jobs:
         test-name:
           - tests::neon_integrations::miner_submit_twice
           - tests::neon_integrations::microblock_integration_test
+          - tests::neon_integrations::microblock_fork_poison_integration_test
           - tests::neon_integrations::size_check_integration_test
           - tests::neon_integrations::cost_voting_integration
           - tests::integrations::integration_test_get_info
@@ -59,7 +60,6 @@ jobs:
           - tests::neon_integrations::antientropy_integration_test
           - tests::neon_integrations::filter_low_fee_tx_integration_test
           - tests::neon_integrations::filter_long_runtime_tx_integration_test
-          - tests::neon_integrations::mining_transactions_is_fair
           - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
           - tests::neon_integrations::block_large_tx_integration_test
           - tests::neon_integrations::microblock_limit_hit_integration_test
@@ -73,6 +73,11 @@ jobs:
           - tests::epoch_205::test_cost_limit_switch_version205
           - tests::epoch_205::test_exact_block_costs
           - tests::epoch_205::bigger_microblock_streams_in_2_05
+          - tests::neon_integrations::test_problematic_txs_are_not_stored
+          - tests::neon_integrations::test_problematic_blocks_are_not_mined
+          - tests::neon_integrations::test_problematic_blocks_are_not_relayed_or_stored
+          - tests::neon_integrations::test_problematic_microblocks_are_not_mined
+          - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored
     steps:
       - uses: actions/checkout@v2
       - name: Download docker image

.github/workflows/ci.yml

@@ -150,14 +150,14 @@ jobs:
           name: ${{ matrix.platform }}
           path: ${{ matrix.platform }}.zip
 
-  call-docker-platforms-workflow:
-    if: ${{ github.event.inputs.tag != '' }}
-    uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master
-    with:
-      tag: ${{ github.event.inputs.tag }}
-    secrets:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+#  call-docker-platforms-workflow:
+#    if: ${{ github.event.inputs.tag != '' }}
+#    uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master
+#    with:
+#      tag: ${{ github.event.inputs.tag }}
+#    secrets:
+#      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+#      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
 
   # Build docker image, tag it with the git tag and `latest` if running on master branch, and publish under the following conditions
   # Will publish if:

CHANGELOG.md

@@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
+## [2.05.0.5.0]
+
+### Changed
+
+- The act of walking the mempool will now cache address nonces in RAM and to a
+  temporary mempool table used for the purpose, instead of unconditionally
+querying them from the chainstate MARF.  This builds upon improvements to mempool
+goodput over 2.05.0.4.0 (#3337).
+- The node and miner implementation has been refactored to remove write-lock
+  contention that can arise when the node's chains-coordinator thread attempts to store and
+process newly-discovered (or newly-mined) blocks, and when the node's relayer
+thread attempts to mine a new block.  In addition, the miner logic has been
+moved to a separate thread in order to avoid starving the relayer thread (which
+must handle block and transaction propagation, as well as block-processing).
+The refactored miner thread will be preemptively terminated and restarted
+by the arrival of new Stacks blocks or burnchain blocks, which further
+prevents the miner from holding open write-locks in the underlying
+chainstate databases when there is new chain data to discover (which would
+invalidate the miner's work anyway).  (#3335).
+
+### Fixed
+
+- Fixed `pow` documentation in Clarity (#3338).
+- Backported unit tests that were omitted in the 2.05.0.3.0 release (#3348).
+
 ## [2.05.0.4.0]
 
 ### Fixed

clarity/src/vm/ast/mod.rs

@@ -151,6 +151,143 @@ mod test {
     use crate::vm::MAX_CALL_STACK_DEPTH;
     use std::collections::HashMap;
 
+    #[derive(PartialEq, Debug)]
+    struct UnitTestTracker {
+        invoked_functions: Vec<(ClarityCostFunction, Vec<u64>)>,
+        invocation_count: u64,
+        cost_addition_count: u64,
+    }
+    impl UnitTestTracker {
+        pub fn new() -> Self {
+            UnitTestTracker {
+                invoked_functions: vec![],
+                invocation_count: 0,
+                cost_addition_count: 0,
+            }
+        }
+    }
+    impl CostTracker for UnitTestTracker {
+        fn compute_cost(
+            &mut self,
+            cost_f: ClarityCostFunction,
+            input: &[u64],
+        ) -> std::result::Result<ExecutionCost, CostErrors> {
+            self.invoked_functions.push((cost_f, input.to_vec()));
+            self.invocation_count += 1;
+            Ok(ExecutionCost::zero())
+        }
+        fn add_cost(&mut self, _cost: ExecutionCost) -> std::result::Result<(), CostErrors> {
+            self.cost_addition_count += 1;
+            Ok(())
+        }
+        fn add_memory(&mut self, _memory: u64) -> std::result::Result<(), CostErrors> {
+            Ok(())
+        }
+        fn drop_memory(&mut self, _memory: u64) {}
+        fn reset_memory(&mut self) {}
+        fn short_circuit_contract_call(
+            &mut self,
+            _contract: &QualifiedContractIdentifier,
+            _function: &ClarityName,
+            _input: &[u64],
+        ) -> Result<bool, CostErrors> {
+            Ok(false)
+        }
+    }
+
+    #[test]
+    fn test_cost_tracking_deep_contracts() {
+        let stack_limit =
+            (AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1) as usize;
+        let exceeds_stack_depth_tuple = format!(
+            "{}u1 {}",
+            "{ a : ".repeat(stack_limit + 1),
+            "} ".repeat(stack_limit + 1)
+        );
+
+        // for deep lists, a test like this works:
+        //   it can assert a limit, that you can also verify
+        //   by disabling `VaryStackDepthChecker` and arbitrarily bumping up the parser lexer limits
+        //   and see that it produces the same result
+        let exceeds_stack_depth_list = format!(
+            "{}u1 {}",
+            "(list ".repeat(stack_limit + 1),
+            ")".repeat(stack_limit + 1)
+        );
+
+        // with old rules, this is just ExpressionStackDepthTooDeep
+        let mut cost_track = UnitTestTracker::new();
+        let err = build_ast_with_rules(
+            &QualifiedContractIdentifier::transient(),
+            &exceeds_stack_depth_list,
+            &mut cost_track,
+            ASTRules::Typical,
+        )
+        .expect_err("Contract should error in parsing");
+
+        let expected_err = ParseErrors::ExpressionStackDepthTooDeep;
+        let expected_list_cost_state = UnitTestTracker {
+            invoked_functions: vec![(ClarityCostFunction::AstParse, vec![500])],
+            invocation_count: 1,
+            cost_addition_count: 1,
+        };
+
+        assert_eq!(&expected_err, &err.err);
+        assert_eq!(expected_list_cost_state, cost_track);
+
+        // with new rules, this is now VaryExpressionStackDepthTooDeep
+        let mut cost_track = UnitTestTracker::new();
+        let err = build_ast_with_rules(
+            &QualifiedContractIdentifier::transient(),
+            &exceeds_stack_depth_list,
+            &mut cost_track,
+            ASTRules::PrecheckSize,
+        )
+        .expect_err("Contract should error in parsing");
+
+        let expected_err = ParseErrors::VaryExpressionStackDepthTooDeep;
+        let expected_list_cost_state = UnitTestTracker {
+            invoked_functions: vec![(ClarityCostFunction::AstParse, vec![500])],
+            invocation_count: 1,
+            cost_addition_count: 1,
+        };
+
+        assert_eq!(&expected_err, &err.err);
+        assert_eq!(expected_list_cost_state, cost_track);
+
+        // you cannot do the same for tuples!
+        // in ASTRules::Typical, this passes
+        let mut cost_track = UnitTestTracker::new();
+        let _ = build_ast_with_rules(
+            &QualifiedContractIdentifier::transient(),
+            &exceeds_stack_depth_tuple,
+            &mut cost_track,
+            ASTRules::Typical,
+        )
+        .expect("Contract should aprse with ASTRules::Typical");
+
+        // this actually won't even error without
+        //  the VaryStackDepthChecker changes.
+        let mut cost_track = UnitTestTracker::new();
+        let err = build_ast_with_rules(
+            &QualifiedContractIdentifier::transient(),
+            &exceeds_stack_depth_tuple,
+            &mut cost_track,
+            ASTRules::PrecheckSize,
+        )
+        .expect_err("Contract should error in parsing with ASTRules::PrecheckSize");
+
+        let expected_err = ParseErrors::VaryExpressionStackDepthTooDeep;
+        let expected_list_cost_state = UnitTestTracker {
+            invoked_functions: vec![(ClarityCostFunction::AstParse, vec![571])],
+            invocation_count: 1,
+            cost_addition_count: 1,
+        };
+
+        assert_eq!(&expected_err, &err.err);
+        assert_eq!(expected_list_cost_state, cost_track);
+    }
+
     #[test]
     fn test_expression_identification_tuples() {
         let progn = "{ a: (+ 1 2 3),

clarity/src/vm/ast/parser/mod.rs

@@ -1059,6 +1059,18 @@ mod test {
         let string_with_multiple_slashes = r#"
         "hello\\\"world"
         "#;
+        let stack_limit =
+            (AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1) as usize;
+        let exceeds_stack_depth_tuple = format!(
+            "{}u1 {}",
+            "{ a : ".repeat(stack_limit + 1),
+            "} ".repeat(stack_limit + 1)
+        );
+        let exceeds_stack_depth_list = format!(
+            "{}u1 {}",
+            "(list ".repeat(stack_limit + 1),
+            ")".repeat(stack_limit + 1)
+        );
 
         assert!(match ast::parser::parse(&split_tokens).unwrap_err().err {
             ParseErrors::SeparatorExpected(_) => true,
@@ -1270,5 +1282,25 @@ mod test {
                 _ => false,
             }
         );
+
+        assert!(match ast::parser::parse(&exceeds_stack_depth_tuple)
+            .unwrap_err()
+            .err
+        {
+            ParseErrors::VaryExpressionStackDepthTooDeep => true,
+            x => {
+                panic!("Got {:?}", &x);
+            }
+        });
+
+        assert!(match ast::parser::parse(&exceeds_stack_depth_list)
+            .unwrap_err()
+            .err
+        {
+            ParseErrors::VaryExpressionStackDepthTooDeep => true,
+            x => {
+                panic!("Got {:?}", &x);
+            }
+        });
     }
 }

clarity/src/vm/docs/mod.rs

@@ -216,7 +216,13 @@ const MOD_API: SimpleFunctionAPI = SimpleFunctionAPI {
 const POW_API: SimpleFunctionAPI = SimpleFunctionAPI {
     name: None,
     signature: "(pow i1 i2)",
-    description: "Returns the result of raising `i1` to the power of `i2`. In the event of an _overflow_, throws a runtime error.",
+    description: "Returns the result of raising `i1` to the power of `i2`. In the event of an _overflow_, throws a runtime error.
+Note: Corner cases are handled with the following rules:
+  * if both `i1` and `i2` are `0`, return `1`
+  * if `i1` is `1`, return `1`
+  * if `i1` is `0`, return `0`
+  * if `i2` is `1`, return `i1`
+  * if `i2` is negative or greater than `u32::MAX`, throw a runtime error",
     example: "(pow 2 3) ;; Returns 8
 (pow 2 2) ;; Returns 4
 (pow 7 1) ;; Returns 7