Farmer checksums

crates/pallet-subspace/src/mock.rs

@@ -50,7 +50,7 @@ use subspace_core_primitives::{
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
-use subspace_farmer_components::sector::{sector_size, SectorMetadata};
+use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::shim::ShimTable;
 use subspace_proof_of_space::Table;
@@ -421,7 +421,7 @@ pub fn create_signed_vote(
 
     for sector_index in iter::from_fn(|| Some(rand::random())) {
         let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadata::encoded_size()];
+        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/sp-lightclient/src/tests.rs

@@ -31,7 +31,7 @@ use subspace_core_primitives::{
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
-use subspace_farmer_components::sector::{sector_size, SectorMetadata};
+use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::Table;
 use subspace_solving::REWARD_SIGNING_CONTEXT;
@@ -159,7 +159,7 @@ fn valid_header(
 
     for sector_index in iter::from_fn(|| Some(rand::random())) {
         let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadata::encoded_size()];
+        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/subspace-archiving/src/archiver.rs

@@ -66,7 +66,7 @@ impl Encode for Segment {
         RecordedHistorySegment::SIZE
     }
 
-    fn encode_to<T: Output + ?Sized>(&self, dest: &mut T) {
+    fn encode_to<O: Output + ?Sized>(&self, dest: &mut O) {
         match self {
             Segment::V0 { items } => {
                 dest.push_byte(0);

crates/subspace-core-primitives/Cargo.toml

@@ -54,6 +54,7 @@ default = [
 embedded-kzg-settings = []
 # Enables some APIs and internal parallelism for KZG
 parallel = [
+    "blake3/rayon",
     "blst_rust/parallel",
     "dep:rayon",
 ]

crates/subspace-core-primitives/src/checksum.rs

@@ -0,0 +1,144 @@
+//! Module containing wrapper for SCALE encoding/decoding with checksum
+
+#[cfg(test)]
+mod tests;
+
+use crate::Blake3Hash;
+use core::mem;
+use parity_scale_codec::{Decode, Encode, EncodeLike, Error, Input, Output};
+
+/// Output wrapper for SCALE codec that will write Blake3 checksum at the end of the encoding
+struct Blake3ChecksumOutput<'a, O>
+where
+    O: Output + ?Sized,
+{
+    output: &'a mut O,
+    hasher: blake3::Hasher,
+}
+
+impl<'a, O> Drop for Blake3ChecksumOutput<'a, O>
+where
+    O: Output + ?Sized,
+{
+    #[inline]
+    fn drop(&mut self) {
+        // Write checksum at the very end of encoding
+        let hash = *self.hasher.finalize().as_bytes();
+        hash.encode_to(self.output);
+    }
+}
+
+impl<'a, O> Output for Blake3ChecksumOutput<'a, O>
+where
+    O: Output + ?Sized,
+{
+    #[inline]
+    fn write(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+        self.output.write(bytes);
+    }
+}
+
+impl<'a, O> Blake3ChecksumOutput<'a, O>
+where
+    O: Output + ?Sized,
+{
+    fn new(output: &'a mut O) -> Self {
+        Self {
+            output,
+            hasher: blake3::Hasher::new(),
+        }
+    }
+}
+
+/// Input wrapper for SCALE codec that will write Blake3 checksum at the end of the encoding
+struct Blake3ChecksumInput<'a, I>
+where
+    I: Input,
+{
+    input: &'a mut I,
+    hasher: blake3::Hasher,
+}
+
+impl<'a, I> Input for Blake3ChecksumInput<'a, I>
+where
+    I: Input,
+{
+    #[inline]
+    fn remaining_len(&mut self) -> Result<Option<usize>, Error> {
+        self.input.remaining_len()
+    }
+
+    #[inline]
+    fn read(&mut self, into: &mut [u8]) -> Result<(), Error> {
+        self.input.read(into)?;
+        self.hasher.update(into);
+        Ok(())
+    }
+}
+
+impl<'a, I> Blake3ChecksumInput<'a, I>
+where
+    I: Input,
+{
+    fn new(output: &'a mut I) -> Self {
+        Self {
+            input: output,
+            hasher: blake3::Hasher::new(),
+        }
+    }
+
+    fn finish(self) -> (Blake3Hash, &'a mut I) {
+        // Compute checksum at the very end of decoding
+        let hash = *self.hasher.finalize().as_bytes();
+        (hash, self.input)
+    }
+}
+
+/// Wrapper data structure that when encoded/decoded will create/check Blake3 checksum
+#[derive(Debug, Clone)]
+pub struct Blake3Checksummed<T>(pub T);
+
+impl<T> Encode for Blake3Checksummed<T>
+where
+    T: Encode,
+{
+    #[inline]
+    fn size_hint(&self) -> usize {
+        self.0.size_hint() + mem::size_of::<Blake3Hash>()
+    }
+
+    #[inline]
+    fn encode_to<O>(&self, dest: &mut O)
+    where
+        O: Output + ?Sized,
+    {
+        self.0.encode_to(&mut Blake3ChecksumOutput::new(dest));
+    }
+
+    #[inline]
+    fn encoded_size(&self) -> usize {
+        self.0.encoded_size() + mem::size_of::<Blake3Hash>()
+    }
+}
+
+impl<T> EncodeLike for Blake3Checksummed<T> where T: EncodeLike {}
+
+impl<T> Decode for Blake3Checksummed<T>
+where
+    T: Decode,
+{
+    #[inline]
+    fn decode<I: Input>(input: &mut I) -> Result<Self, Error> {
+        let mut input = Blake3ChecksumInput::new(input);
+        let data = T::decode(&mut input)?;
+        let (actual_hash, input) = input.finish();
+        let expected_hash = Blake3Hash::decode(input)?;
+
+        if actual_hash == expected_hash {
+            Ok(Self(data))
+        } else {
+            Err(Error::from("Checksum mismatch"))
+        }
+    }
+}

crates/subspace-core-primitives/src/checksum/tests.rs

@@ -0,0 +1,30 @@
+use super::Blake3Checksummed;
+use crate::Blake3Hash;
+use parity_scale_codec::{Decode, Encode};
+use rand::prelude::*;
+use std::mem;
+
+#[test]
+fn basic() {
+    let random_bytes = random::<[u8; 64]>();
+
+    let plain_encoding = random_bytes.encode();
+    let checksummed_encoding = Blake3Checksummed(random_bytes).encode();
+
+    // Encoding is extended with checksum
+    assert_eq!(
+        plain_encoding.len() + mem::size_of::<Blake3Hash>(),
+        checksummed_encoding.len()
+    );
+
+    // Decoding succeeds
+    let Blake3Checksummed(decoded_random_bytes) =
+        Blake3Checksummed::<[u8; 64]>::decode(&mut checksummed_encoding.as_slice()).unwrap();
+    // Decodes to original data
+    assert_eq!(random_bytes, decoded_random_bytes);
+
+    // Non-checksummed encoding fails to decode
+    assert!(Blake3Checksummed::<[u8; 64]>::decode(&mut plain_encoding.as_slice()).is_err());
+    // Incorrectly checksummed data fails to decode
+    assert!(Blake3Checksummed::<[u8; 32]>::decode(&mut random::<[u8; 64]>().as_ref()).is_err());
+}

crates/subspace-core-primitives/src/crypto.rs

@@ -89,6 +89,14 @@ pub fn blake3_hash(data: &[u8]) -> Blake3Hash {
     *blake3::hash(data).as_bytes()
 }
 
+/// BLAKE3 hashing of a single value in parallel (only useful for large values well above 128kiB).
+#[cfg(feature = "parallel")]
+pub fn blake3_hash_parallel(data: &[u8]) -> Blake3Hash {
+    let mut state = blake3::Hasher::new();
+    state.update_rayon(data);
+    *state.finalize().as_bytes()
+}
+
 /// BLAKE3 hashing of a list of values.
 pub fn blake3_hash_list(data: &[&[u8]]) -> Blake3Hash {
     let mut state = blake3::Hasher::new();

crates/subspace-core-primitives/src/lib.rs

@@ -29,6 +29,7 @@
     step_trait
 )]
 
+pub mod checksum;
 pub mod crypto;
 pub mod objects;
 mod pieces;
@@ -122,7 +123,7 @@ impl Randomness {
     /// Derive global slot challenge from global randomness.
     // TODO: Separate type for global challenge
     pub fn derive_global_challenge(&self, slot: SlotNumber) -> Blake2b256Hash {
-        crypto::blake2b_256_hash_list(&[&self.0, &slot.to_le_bytes()])
+        blake2b_256_hash_list(&[&self.0, &slot.to_le_bytes()])
     }
 }
 

crates/subspace-farmer-components/Cargo.toml

@@ -21,6 +21,7 @@ backoff = { version = "0.4.0", features = ["futures", "tokio"] }
 bitvec = "1.0.1"
 fs2 = "0.4.3"
 futures = "0.3.28"
+hex = "0.4.3"
 libc = "0.2.146"
 lru = "0.10.0"
 parity-scale-codec = "3.6.3"

crates/subspace-farmer-components/benches/auditing.rs

@@ -18,7 +18,9 @@ use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::file_ext::FileExt;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy, PlottedSector};
-use subspace_farmer_components::sector::{sector_size, SectorContentsMap, SectorMetadata};
+use subspace_farmer_components::sector::{
+    sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
+};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::chia::ChiaTable;
 use subspace_proof_of_space::Table;
@@ -93,12 +95,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             pieces_in_sector,
         )
         .unwrap();
-        let sector_metadata = SectorMetadata {
+        let sector_metadata = SectorMetadataChecksummed::from(SectorMetadata {
             sector_index,
             pieces_in_sector,
             s_bucket_sizes: sector_contents_map.s_bucket_sizes(),
             history_size: farmer_protocol_info.history_size,
-        };
+        });
 
         (
             PlottedSector {
@@ -113,7 +115,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         println!("Plotting one sector...");
 
         let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadata::encoded_size()];
+        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/subspace-farmer-components/benches/plotting.rs

@@ -9,7 +9,7 @@ use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::{HistorySize, PublicKey, Record, RecordedHistorySegment};
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
-use subspace_farmer_components::sector::{sector_size, SectorMetadata};
+use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::chia::ChiaTable;
 use subspace_proof_of_space::Table;
@@ -59,7 +59,7 @@ fn criterion_benchmark(c: &mut Criterion) {
 
     let sector_size = sector_size(pieces_in_sector);
     let mut sector_bytes = vec![0; sector_size];
-    let mut sector_metadata_bytes = vec![0; SectorMetadata::encoded_size()];
+    let mut sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
 
     let mut group = c.benchmark_group("plotting");
     group.throughput(Throughput::Bytes(sector_size as u64));

crates/subspace-farmer-components/benches/proving.rs

@@ -18,7 +18,9 @@ use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::file_ext::FileExt;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy, PlottedSector};
-use subspace_farmer_components::sector::{sector_size, SectorContentsMap, SectorMetadata};
+use subspace_farmer_components::sector::{
+    sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
+};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::chia::ChiaTable;
 use subspace_proof_of_space::Table;
@@ -95,12 +97,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             pieces_in_sector,
         )
         .unwrap();
-        let sector_metadata = SectorMetadata {
+        let sector_metadata = SectorMetadataChecksummed::from(SectorMetadata {
             sector_index,
             pieces_in_sector,
             s_bucket_sizes: sector_contents_map.s_bucket_sizes(),
             history_size: farmer_protocol_info.history_size,
-        };
+        });
 
         (
             PlottedSector {
@@ -115,7 +117,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         println!("Plotting one sector...");
 
         let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadata::encoded_size()];
+        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,