Implement Trusted Vector Preallocation (#1920)

* Implement SafePreallocate. Resolves #1880

* Add proptests for SafePreallocate

* Apply suggestions from code review

Comments which did not include replacement code will be addressed in a follow-up commit.

Co-authored-by: teor <[email protected]>

* Rename [Safe-> Trusted]Allocate. Add doc and tests

Add tests to show that the largest allowed vec under TrustedPreallocate
is small enough to fit in a Zcash block/message (depending on type).
Add doc comments to all TrustedPreallocate test cases.
Tighten bounds on max_trusted_alloc for some types.

Note - this commit does NOT include TrustedPreallocate
impls for JoinSplitData, String, and Script.
These impls will be added in a follow up commit

* Implement SafePreallocate. Resolves #1880

* Add proptests for SafePreallocate

* Apply suggestions from code review

Comments which did not include replacement code will be addressed in a follow-up commit.

Co-authored-by: teor <[email protected]>

* Rename [Safe-> Trusted]Allocate. Add doc and tests

Add tests to show that the largest allowed vec under TrustedPreallocate
is small enough to fit in a Zcash block/message (depending on type).
Add doc comments to all TrustedPreallocate test cases.
Tighten bounds on max_trusted_alloc for some types.

Note - this commit does NOT include TrustedPreallocate
impls for JoinSplitData, String, and Script.
These impls will be added in a follow up commit

* Impl TrustedPreallocate for Joinsplit

* Impl ZcashDeserialize for Vec<u8>

* Arbitrary, TrustedPreallocate, Serialize, and tests for Spend<SharedAnchor>

Co-authored-by: teor <[email protected]>

zebra-chain/src/block.rs

@@ -21,10 +21,17 @@ pub use hash::Hash;
 pub use header::BlockTimeError;
 pub use header::{CountedHeader, Header};
 pub use height::Height;
+pub use serialize::MAX_BLOCK_BYTES;
 
 use serde::{Deserialize, Serialize};
 
-use crate::{fmt::DisplayToDebug, parameters::Network, transaction::Transaction, transparent};
+use crate::{
+    fmt::DisplayToDebug,
+    parameters::Network,
+    serialization::{TrustedPreallocate, MAX_PROTOCOL_MESSAGE_LEN},
+    transaction::Transaction,
+    transparent,
+};
 
 /// A Zcash block, containing a header and a list of transactions.
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
@@ -80,3 +87,63 @@ impl<'a> From<&'a Block> for Hash {
         (&block.header).into()
     }
 }
+/// A serialized Block hash takes 32 bytes
+const BLOCK_HASH_SIZE: u64 = 32;
+/// The maximum number of hashes in a valid Zcash protocol message.
+impl TrustedPreallocate for Hash {
+    fn max_allocation() -> u64 {
+        // Every vector type requires a length field of at least one byte for de/serialization.
+        // Since a block::Hash takes 32 bytes, we can never receive more than (MAX_PROTOCOL_MESSAGE_LEN - 1) / 32 hashes in a single message
+        ((MAX_PROTOCOL_MESSAGE_LEN - 1) as u64) / BLOCK_HASH_SIZE
+    }
+}
+
+#[cfg(test)]
+mod test_trusted_preallocate {
+    use super::{Hash, BLOCK_HASH_SIZE, MAX_PROTOCOL_MESSAGE_LEN};
+    use crate::serialization::{TrustedPreallocate, ZcashSerialize};
+    use proptest::prelude::*;
+    use std::convert::TryInto;
+    proptest! {
+        #![proptest_config(ProptestConfig::with_cases(10_000))]
+        /// Verify that the serialized size of a block hash used to calculate the allocation limit is correct
+        #[test]
+        fn block_hash_size_is_correct(hash in Hash::arbitrary()) {
+            let serialized = hash.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+            prop_assert!(serialized.len() as u64 == BLOCK_HASH_SIZE);
+        }
+    }
+    proptest! {
+
+        #![proptest_config(ProptestConfig::with_cases(200))]
+
+        /// Verify that...
+        /// 1. The smallest disallowed vector of `Hash`s is too large to send via the Zcash Wire Protocol
+        /// 2. The largest allowed vector is small enough to fit in a legal Zcash Wire Protocol message
+        #[test]
+        fn block_hash_max_allocation(hash in Hash::arbitrary_with(())) {
+            let max_allocation: usize = Hash::max_allocation().try_into().unwrap();
+            let mut smallest_disallowed_vec = Vec::with_capacity(max_allocation + 1);
+            for _ in 0..(Hash::max_allocation()+1) {
+                smallest_disallowed_vec.push(hash);
+            }
+
+            let smallest_disallowed_serialized = smallest_disallowed_vec.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+            // Check that our smallest_disallowed_vec is only one item larger than the limit
+            prop_assert!(((smallest_disallowed_vec.len() - 1) as u64) == Hash::max_allocation());
+            // Check that our smallest_disallowed_vec is too big to send as a protocol message
+            prop_assert!(smallest_disallowed_serialized.len() > MAX_PROTOCOL_MESSAGE_LEN);
+
+            // Create largest_allowed_vec by removing one element from smallest_disallowed_vec without copying (for efficiency)
+            smallest_disallowed_vec.pop();
+            let largest_allowed_vec = smallest_disallowed_vec;
+            let largest_allowed_serialized = largest_allowed_vec.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+
+            // Check that our largest_allowed_vec contains the maximum number of hashes
+            prop_assert!((largest_allowed_vec.len() as u64) == Hash::max_allocation());
+            // Check that our largest_allowed_vec is small enough to send as a protocol message
+            prop_assert!(largest_allowed_serialized.len() <= MAX_PROTOCOL_MESSAGE_LEN);
+
+        }
+    }
+}

zebra-chain/src/block/header.rs

@@ -1,7 +1,12 @@
+use std::usize;
+
 use chrono::{DateTime, Duration, Utc};
 use thiserror::Error;
 
-use crate::work::{difficulty::CompactDifficulty, equihash::Solution};
+use crate::{
+    serialization::{TrustedPreallocate, MAX_PROTOCOL_MESSAGE_LEN},
+    work::{difficulty::CompactDifficulty, equihash::Solution},
+};
 
 use super::{merkle, Hash, Height};
 
@@ -118,3 +123,79 @@ pub struct CountedHeader {
     pub header: Header,
     pub transaction_count: usize,
 }
+
+/// The serialized size of a Zcash block header.
+///
+/// Includes the equihash input, 32-byte nonce, 3-byte equihash length field, and equihash solution.
+const BLOCK_HEADER_LENGTH: usize =
+    crate::work::equihash::Solution::INPUT_LENGTH + 32 + 3 + crate::work::equihash::SOLUTION_SIZE;
+
+/// The minimum size for a serialized CountedHeader.
+///
+/// A CountedHeader has BLOCK_HEADER_LENGTH bytes + 1 or more bytes for the transaction count
+const MIN_COUNTED_HEADER_LEN: usize = BLOCK_HEADER_LENGTH + 1;
+impl TrustedPreallocate for CountedHeader {
+    fn max_allocation() -> u64 {
+        // Every vector type requires a length field of at least one byte for de/serialization.
+        // Therefore, we can never receive more than (MAX_PROTOCOL_MESSAGE_LEN - 1) / MIN_COUNTED_HEADER_LEN counted headers in a single message
+        ((MAX_PROTOCOL_MESSAGE_LEN - 1) / MIN_COUNTED_HEADER_LEN) as u64
+    }
+}
+
+#[cfg(test)]
+mod test_trusted_preallocate {
+    use super::{CountedHeader, Header, MAX_PROTOCOL_MESSAGE_LEN, MIN_COUNTED_HEADER_LEN};
+    use crate::serialization::{TrustedPreallocate, ZcashSerialize};
+    use proptest::prelude::*;
+    use std::convert::TryInto;
+    proptest! {
+
+        #![proptest_config(ProptestConfig::with_cases(10_000))]
+
+        /// Confirm that each counted header takes at least COUNTED_HEADER_LEN bytes when serialized.
+        /// This verifies that our calculated `TrustedPreallocate::max_allocation()` is indeed an upper bound.
+        #[test]
+        fn counted_header_min_length(header in Header::arbitrary_with(()), transaction_count in (0..std::u32::MAX)) {
+            let header = CountedHeader {
+                header,
+                transaction_count: transaction_count.try_into().expect("Must run test on platform with at least 32 bit address space"),
+            };
+            let serialized_header = header.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+            prop_assert!(serialized_header.len() >= MIN_COUNTED_HEADER_LEN)
+        }
+    }
+    proptest! {
+        #![proptest_config(ProptestConfig::with_cases(100))]
+        /// Verify that...
+        /// 1. The smallest disallowed vector of `CountedHeaders`s is too large to send via the Zcash Wire Protocol
+        /// 2. The largest allowed vector is small enough to fit in a legal Zcash Wire Protocol message
+        #[test]
+        fn counted_header_max_allocation(header in Header::arbitrary_with(())) {
+            let header = CountedHeader {
+                header,
+                transaction_count: 0,
+            };
+            let max_allocation: usize = CountedHeader::max_allocation().try_into().unwrap();
+            let mut smallest_disallowed_vec = Vec::with_capacity(max_allocation + 1);
+            for _ in 0..(CountedHeader::max_allocation()+1) {
+                smallest_disallowed_vec.push(header.clone());
+            }
+            let smallest_disallowed_serialized = smallest_disallowed_vec.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+            // Check that our smallest_disallowed_vec is only one item larger than the limit
+            prop_assert!(((smallest_disallowed_vec.len() - 1) as u64) == CountedHeader::max_allocation());
+            // Check that our smallest_disallowed_vec is too big to send as a protocol message
+            prop_assert!(smallest_disallowed_serialized.len() > MAX_PROTOCOL_MESSAGE_LEN);
+
+
+            // Create largest_allowed_vec by removing one element from smallest_disallowed_vec without copying (for efficiency)
+            smallest_disallowed_vec.pop();
+            let largest_allowed_vec = smallest_disallowed_vec;
+            let largest_allowed_serialized = largest_allowed_vec.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+
+            // Check that our largest_allowed_vec contains the maximum number of CountedHeaders
+            prop_assert!((largest_allowed_vec.len() as u64) == CountedHeader::max_allocation());
+            // Check that our largest_allowed_vec is small enough to send as a protocol message
+            prop_assert!(largest_allowed_serialized.len() <= MAX_PROTOCOL_MESSAGE_LEN);
+        }
+    }
+}

zebra-chain/src/sapling/arbitrary.rs

@@ -3,7 +3,9 @@ use proptest::{arbitrary::any, array, collection::vec, prelude::*};
 
 use crate::primitives::Groth16Proof;
 
-use super::{keys, note, tree, NoteCommitment, Output, PerSpendAnchor, Spend, ValueCommitment};
+use super::{
+    keys, note, tree, NoteCommitment, Output, PerSpendAnchor, SharedAnchor, Spend, ValueCommitment,
+};
 
 impl Arbitrary for Spend<PerSpendAnchor> {
     type Parameters = ();
@@ -36,6 +38,34 @@ impl Arbitrary for Spend<PerSpendAnchor> {
     type Strategy = BoxedStrategy<Self>;
 }
 
+impl Arbitrary for Spend<SharedAnchor> {
+    type Parameters = ();
+
+    fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
+        (
+            any::<note::Nullifier>(),
+            array::uniform32(any::<u8>()),
+            any::<Groth16Proof>(),
+            vec(any::<u8>(), 64),
+        )
+            .prop_map(|(nullifier, rpk_bytes, proof, sig_bytes)| Self {
+                per_spend_anchor: (),
+                cv: ValueCommitment(AffinePoint::identity()),
+                nullifier,
+                rk: redjubjub::VerificationKeyBytes::from(rpk_bytes),
+                zkproof: proof,
+                spend_auth_sig: redjubjub::Signature::from({
+                    let mut b = [0u8; 64];
+                    b.copy_from_slice(sig_bytes.as_slice());
+                    b
+                }),
+            })
+            .boxed()
+    }
+
+    type Strategy = BoxedStrategy<Self>;
+}
+
 impl Arbitrary for Output {
     type Parameters = ();
 

zebra-chain/src/sapling/output.rs

@@ -1,8 +1,11 @@
 use std::io;
 
 use crate::{
+    block::MAX_BLOCK_BYTES,
     primitives::Groth16Proof,
-    serialization::{serde_helpers, SerializationError, ZcashDeserialize, ZcashSerialize},
+    serialization::{
+        serde_helpers, SerializationError, TrustedPreallocate, ZcashDeserialize, ZcashSerialize,
+    },
 };
 
 use super::{commitment, keys, note};
@@ -75,3 +78,73 @@ impl ZcashDeserialize for Output {
         })
     }
 }
+/// An output contains: a 32 byte cv, a 32 byte cmu, a 32 byte ephemeral key
+/// a 580 byte encCiphertext, an 80 byte outCiphertext, and a 192 byte zkproof
+/// [ps]: https://zips.z.cash/protocol/protocol.pdf#outputencoding
+const OUTPUT_SIZE: u64 = 32 + 32 + 32 + 580 + 80 + 192;
+
+/// The maximum number of outputs in a valid Zcash on-chain transaction.
+///
+/// If a transaction contains more outputs than can fit in maximally large block, it might be
+/// valid on the network and in the mempool, but it can never be mined into a block. So
+/// rejecting these large edge-case transactions can never break consensus
+impl TrustedPreallocate for Output {
+    fn max_allocation() -> u64 {
+        // Since a serialized Vec<Output> uses at least one byte for its length,
+        // the max allocation can never exceed (MAX_BLOCK_BYTES - 1) / OUTPUT_SIZE
+        (MAX_BLOCK_BYTES - 1) / OUTPUT_SIZE
+    }
+}
+
+#[cfg(test)]
+mod test_trusted_preallocate {
+    use super::{Output, MAX_BLOCK_BYTES, OUTPUT_SIZE};
+    use crate::serialization::{TrustedPreallocate, ZcashSerialize};
+    use proptest::prelude::*;
+    use std::convert::TryInto;
+
+    proptest! {
+        #![proptest_config(ProptestConfig::with_cases(10_000))]
+
+        /// Confirm that each output takes exactly OUTPUT_SIZE bytes when serialized.
+        /// This verifies that our calculated `TrustedPreallocate::max_allocation()` is indeed an upper bound.
+        #[test]
+        fn output_size_is_small_enough(output in Output::arbitrary_with(())) {
+            let serialized = output.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+            prop_assert!(serialized.len() as u64 == OUTPUT_SIZE)
+        }
+
+    }
+    proptest! {
+        #![proptest_config(ProptestConfig::with_cases(100))]
+        /// Verify that...
+        /// 1. The smallest disallowed vector of `Outputs`s is too large to fit in a Zcash block
+        /// 2. The largest allowed vector is small enough to fit in a legal Zcash block
+        #[test]
+        fn output_max_allocation_is_big_enough(output in Output::arbitrary_with(())) {
+
+            let max_allocation: usize = Output::max_allocation().try_into().unwrap();
+            let mut smallest_disallowed_vec = Vec::with_capacity(max_allocation + 1);
+            for _ in 0..(Output::max_allocation()+1) {
+                smallest_disallowed_vec.push(output.clone());
+            }
+            let smallest_disallowed_serialized = smallest_disallowed_vec.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+            // Check that our smallest_disallowed_vec is only one item larger than the limit
+            prop_assert!(((smallest_disallowed_vec.len() - 1) as u64) == Output::max_allocation());
+            // Check that our smallest_disallowed_vec is too big to be included in a valid block
+            // Note that a serialized block always includes at least one byte for the number of transactions,
+            // so any serialized Vec<Output> at least MAX_BLOCK_BYTES long is too large to fit in a block.
+            prop_assert!((smallest_disallowed_serialized.len() as u64) >= MAX_BLOCK_BYTES);
+
+            // Create largest_allowed_vec by removing one element from smallest_disallowed_vec without copying (for efficiency)
+            smallest_disallowed_vec.pop();
+            let largest_allowed_vec = smallest_disallowed_vec;
+            let largest_allowed_serialized = largest_allowed_vec.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+
+            // Check that our largest_allowed_vec contains the maximum number of Outputs
+            prop_assert!((largest_allowed_vec.len() as u64) == Output::max_allocation());
+            // Check that our largest_allowed_vec is small enough to fit in a Zcash block.
+            prop_assert!((largest_allowed_serialized.len() as u64) < MAX_BLOCK_BYTES);
+        }
+    }
+}