From eddf4cb979dd9285f2b76a357927874fa48c5b33 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 29 Jan 2024 14:20:34 +0200 Subject: [PATCH 01/77] WIP --- fuel-tx/Cargo.toml | 1 + fuel-tx/src/transaction/policies.rs | 1 + fuel-tx/src/transaction/types/input.rs | 1 + fuel-tx/src/transaction/types/input/coin.rs | 3 + .../src/transaction/types/input/contract.rs | 5 + .../src/transaction/types/input/message.rs | 5 + fuel-tx/src/transaction/types/mint.rs | 4 + fuel-tx/src/transaction/types/output.rs | 28 +- .../src/transaction/types/output/contract.rs | 3 + fuel-tx/src/transaction/types/script.rs | 3 + fuel-tx/src/transaction/types/utxo_id.rs | 1 + fuel-types/Cargo.toml | 1 + fuel-types/src/array_types.rs | 2 + fuel-types/src/compressed.rs | 443 ++++++++++++++++++ 14 files changed, 488 insertions(+), 13 deletions(-) create mode 100644 fuel-types/src/compressed.rs diff --git a/fuel-tx/Cargo.toml b/fuel-tx/Cargo.toml index 1a347cca6f..31112062f3 100644 --- a/fuel-tx/Cargo.toml +++ b/fuel-tx/Cargo.toml @@ -28,6 +28,7 @@ serde_json = { version = "1.0", default-features = false, features = ["alloc"], strum = { version = "0.24", default-features = false, optional = true } strum_macros = { version = "0.24", optional = true } wasm-bindgen = { version = "0.2.88", optional = true } +fuel-core-compression = { path = "../../fuel-core/crates/compression" } [dev-dependencies] bincode = { workspace = true } diff --git a/fuel-tx/src/transaction/policies.rs b/fuel-tx/src/transaction/policies.rs index cf95002cd0..0e6f59723e 100644 --- a/fuel-tx/src/transaction/policies.rs +++ b/fuel-tx/src/transaction/policies.rs @@ -83,6 +83,7 @@ pub const POLICIES_NUMBER: usize = PoliciesBits::all().bits().count_ones() as us #[derive(Clone, Copy, Default, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct Policies { /// A bitmask that indicates what policies are set. bits: PoliciesBits, diff --git a/fuel-tx/src/transaction/types/input.rs b/fuel-tx/src/transaction/types/input.rs index df07d1fdbb..5402d3d5b4 100644 --- a/fuel-tx/src/transaction/types/input.rs +++ b/fuel-tx/src/transaction/types/input.rs @@ -181,6 +181,7 @@ where #[derive(Debug, Clone, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[non_exhaustive] pub enum Input { CoinSigned(CoinSigned), diff --git a/fuel-tx/src/transaction/types/input/coin.rs b/fuel-tx/src/transaction/types/input/coin.rs index f8896b4bb8..8a32c592c4 100644 --- a/fuel-tx/src/transaction/types/input/coin.rs +++ b/fuel-tx/src/transaction/types/input/coin.rs @@ -97,13 +97,16 @@ impl CoinSpecification for Full { #[derivative(Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct Coin where Specification: CoinSpecification, { pub utxo_id: UtxoId, + #[da_compress(registry = "Address")] pub owner: Address, pub amount: Word, + #[da_compress(registry = "AssetId")] pub asset_id: AssetId, pub tx_pointer: TxPointer, #[derivative(Debug(format_with = "fmt_as_field"))] diff --git a/fuel-tx/src/transaction/types/input/contract.rs b/fuel-tx/src/transaction/types/input/contract.rs index 8fa1bfb1c7..178c07a702 100644 --- a/fuel-tx/src/transaction/types/input/contract.rs +++ b/fuel-tx/src/transaction/types/input/contract.rs @@ -15,12 +15,17 @@ use fuel_types::{ #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen(js_name = InputContract))] pub struct Contract { pub utxo_id: UtxoId, + #[da_compress(skip)] pub balance_root: Bytes32, + #[da_compress(skip)] pub state_root: Bytes32, + #[da_compress(skip)] pub tx_pointer: TxPointer, + #[da_compress(skip)] pub contract_id: ContractId, } diff --git a/fuel-tx/src/transaction/types/input/message.rs b/fuel-tx/src/transaction/types/input/message.rs index 34619bc033..2906c63422 100644 --- a/fuel-tx/src/transaction/types/input/message.rs +++ b/fuel-tx/src/transaction/types/input/message.rs @@ -65,6 +65,7 @@ pub mod specifications { /// is not consumed and can be used later until successful execution. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + #[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct MessageData(core::marker::PhantomData); impl MessageSpecification for MessageData { @@ -144,19 +145,23 @@ pub mod specifications { #[derivative(Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct Message where Specification: MessageSpecification, { /// The sender from the L1 chain. + #[da_compress(registry = "Address")] pub sender: Address, /// The receiver on the `Fuel` chain. + #[da_compress(registry = "Address")] pub recipient: Address, pub amount: Word, pub nonce: Nonce, #[derivative(Debug(format_with = "fmt_as_field"))] pub witness_index: Specification::Witness, #[derivative(Debug(format_with = "fmt_as_field"))] + #[da_compress(skip)] pub predicate_gas_used: Specification::PredicateGasUsed, #[derivative(Debug(format_with = "fmt_as_field"))] pub data: Specification::Data, diff --git a/fuel-tx/src/transaction/types/mint.rs b/fuel-tx/src/transaction/types/mint.rs index b4b961b0b5..f0c30aade8 100644 --- a/fuel-tx/src/transaction/types/mint.rs +++ b/fuel-tx/src/transaction/types/mint.rs @@ -50,11 +50,13 @@ impl MintMetadata { #[derive(Default, Debug, Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[canonical(prefix = TransactionRepr::Mint)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derivative(Eq, PartialEq, Hash)] pub struct Mint { /// The location of the transaction in the block. + #[da_compress(skip)] pub(crate) tx_pointer: TxPointer, /// The `Input::Contract` that assets are minted to. pub(crate) input_contract: input::contract::Contract, @@ -63,10 +65,12 @@ pub struct Mint { /// The amount of funds minted. pub(crate) mint_amount: Word, /// The asset IDs corresponding to the minted amount. + #[da_compress(registry = "AssetId")] pub(crate) mint_asset_id: AssetId, #[cfg_attr(feature = "serde", serde(skip))] #[derivative(PartialEq = "ignore", Hash = "ignore")] #[canonical(skip)] + #[da_compress(skip)] pub(crate) metadata: Option, } diff --git a/fuel-tx/src/transaction/types/output.rs b/fuel-tx/src/transaction/types/output.rs index 211a082b84..8e1bac754a 100644 --- a/fuel-tx/src/transaction/types/output.rs +++ b/fuel-tx/src/transaction/types/output.rs @@ -1,9 +1,6 @@ use fuel_crypto::Hasher; use fuel_types::{ - canonical::{ - Deserialize, - Serialize, - }, + canonical, Address, AssetId, Bytes32, @@ -23,7 +20,8 @@ pub use repr::OutputRepr; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[derive(Deserialize, Serialize)] +#[derive(canonical::Deserialize, canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[non_exhaustive] pub enum Output { Coin { @@ -36,13 +34,17 @@ pub enum Output { Change { to: Address, + #[da_compress(skip)] amount: Word, asset_id: AssetId, }, Variable { + #[da_compress(skip)] to: Address, + #[da_compress(skip)] amount: Word, + #[da_compress(skip)] asset_id: AssetId, }, @@ -199,7 +201,7 @@ impl Output { Output::Contract(contract) => contract.prepare_sign(), Output::Change { amount, .. } => { - mem::take(amount); + *amount = 0; } Output::Variable { @@ -208,9 +210,9 @@ impl Output { asset_id, .. } => { - mem::take(to); - mem::take(amount); - mem::take(asset_id); + *to = Address::default(); + *amount = 0; + *asset_id = AssetId::default(); } _ => (), @@ -221,7 +223,7 @@ impl Output { pub fn prepare_init_script(&mut self) { match self { Output::Change { amount, .. } => { - mem::take(amount); + *amount = 0; } Output::Variable { @@ -229,9 +231,9 @@ impl Output { amount, asset_id, } => { - mem::take(to); - mem::take(amount); - mem::take(asset_id); + *to = Address::default(); + *amount = 0; + *asset_id = AssetId::default(); } _ => (), diff --git a/fuel-tx/src/transaction/types/output/contract.rs b/fuel-tx/src/transaction/types/output/contract.rs index 6e9e3e831e..d1c6ad141d 100644 --- a/fuel-tx/src/transaction/types/output/contract.rs +++ b/fuel-tx/src/transaction/types/output/contract.rs @@ -8,13 +8,16 @@ use fuel_types::Bytes32; #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen(js_name = OutputContract))] pub struct Contract { /// Index of input contract. pub input_index: u8, /// Root of amount of coins owned by contract after transaction execution. + #[da_compress(skip)] pub balance_root: Bytes32, /// State root of contract after transaction execution. + #[da_compress(skip)] pub state_root: Bytes32, } diff --git a/fuel-tx/src/transaction/types/script.rs b/fuel-tx/src/transaction/types/script.rs index 869b401999..e66d8ea1f7 100644 --- a/fuel-tx/src/transaction/types/script.rs +++ b/fuel-tx/src/transaction/types/script.rs @@ -55,11 +55,13 @@ pub(crate) struct ScriptMetadata { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[canonical(prefix = TransactionRepr::Script)] #[derivative(Eq, PartialEq, Hash, Debug)] pub struct Script { pub(crate) script_gas_limit: Word, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] + #[da_compress(registry = "ScriptCode")] pub(crate) script: Vec, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] pub(crate) script_data: Vec, @@ -71,6 +73,7 @@ pub struct Script { #[cfg_attr(feature = "serde", serde(skip))] #[derivative(PartialEq = "ignore", Hash = "ignore")] #[canonical(skip)] + #[da_compress(skip)] pub(crate) metadata: Option, } diff --git a/fuel-tx/src/transaction/types/utxo_id.rs b/fuel-tx/src/transaction/types/utxo_id.rs index 8e1a671325..e089e4e104 100644 --- a/fuel-tx/src/transaction/types/utxo_id.rs +++ b/fuel-tx/src/transaction/types/utxo_id.rs @@ -21,6 +21,7 @@ use rand::{ #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] +#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct UtxoId { /// transaction id tx_id: TxId, diff --git a/fuel-types/Cargo.toml b/fuel-types/Cargo.toml index 3acae8bdf8..7695a0c818 100644 --- a/fuel-types/Cargo.toml +++ b/fuel-types/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } description = "Atomic types of the FuelVM." [dependencies] +fuel-core-compression = { path = "../../fuel-core/crates/compression" } fuel-derive = { workspace = true } hex = { version = "0.4", default-features = false, optional = true } rand = { version = "0.8", default-features = false, optional = true } diff --git a/fuel-types/src/array_types.rs b/fuel-types/src/array_types.rs index e63abd8cc8..d6edf80685 100644 --- a/fuel-types/src/array_types.rs +++ b/fuel-types/src/array_types.rs @@ -38,6 +38,7 @@ macro_rules! key { #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, )] + #[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct $i([u8; $s]); key_methods!($i, $s); @@ -60,6 +61,7 @@ macro_rules! key_with_big_array { #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, )] + #[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct $i([u8; $s]); key_methods!($i, $s); diff --git a/fuel-types/src/compressed.rs b/fuel-types/src/compressed.rs new file mode 100644 index 0000000000..85910949fa --- /dev/null +++ b/fuel-types/src/compressed.rs @@ -0,0 +1,443 @@ +//! Compressed encoding and decoding of Fuel types. +//! +//! This is an extension of the canonical encoding. + +#[cfg(feature = "alloc")] +use alloc::vec::Vec; +use core::fmt; + +use core::mem::MaybeUninit; +pub use fuel_derive::{ + Deserialize, + Serialize, +}; + +/// Allows serialize the type into the `Output`. +pub trait SerializeCompact { + /// Size of the static part of the serialized object, in bytes. + fn size_static(&self) -> usize; + + /// Size of the dynamic part, in bytes. + fn size_dynamic(&self) -> usize; + + /// Total size of the serialized object, in bytes. + fn size(&self) -> usize { + self.size_static().saturating_add(self.size_dynamic()) + } + + /// Encodes `Self` into the `buffer`. + /// + /// It is better to not implement this function directly, instead implement + /// `encode_static` and `encode_dynamic`. + fn encode(&self, buffer: &mut O) -> Result<(), Error> { + self.encode_static(buffer)?; + self.encode_dynamic(buffer) + } + + /// Encodes staticly-sized part of `Self`. + fn encode_static(&self, buffer: &mut O) -> Result<(), Error>; + + /// Encodes dynamically-sized part of `Self`. + /// The default implementation does nothing. Dynamically-sized contains should + /// override this. + fn encode_dynamic(&self, _buffer: &mut O) -> Result<(), Error> { + Ok(()) + } + + /// Encodes `Self` into bytes vector. Required known size. + #[cfg(feature = "alloc")] + fn to_bytes(&self) -> Vec { + let mut vec = Vec::with_capacity(self.size()); + self.encode(&mut vec).expect("Unable to encode self"); + vec + } +} + + +/// Allows deserialize the type from the `Input`. +pub trait DeserializeCompact: Sized { + /// Decodes `Self` from the `buffer`. + /// + /// It is better to not implement this function directly, instead implement + /// `decode_static` and `decode_dynamic`. + fn decode(buffer: &mut I) -> Result { + let mut object = Self::decode_static(buffer)?; + object.decode_dynamic(buffer)?; + Ok(object) + } + + /// Decodes static part of `Self` from the `buffer`. + fn decode_static(buffer: &mut I) -> Result; + + /// Decodes dynamic part of the information from the `buffer` to fill `Self`. + /// The default implementation does nothing. Dynamically-sized contains should + /// override this. + fn decode_dynamic( + &mut self, + _buffer: &mut I, + ) -> Result<(), Error> { + Ok(()) + } + + /// Helper method for deserializing `Self` from bytes. + fn from_bytes(mut buffer: &[u8]) -> Result { + Self::decode(&mut buffer) + } +} + + +macro_rules! impl_for_primitives { + ($t:ident, $unpadded:literal) => { + impl SerializeCompact for $t { + #[inline(always)] + fn size_static(&self) -> usize { + aligned_size(::core::mem::size_of::<$t>()) + } + + #[inline(always)] + fn size_dynamic(&self) -> usize { + 0 + } + + #[inline(always)] + fn encode_static( + &self, + buffer: &mut O, + ) -> Result<(), Error> { + // Primitive types are zero-padded on left side to a 8-byte boundary. + // The resulting value is always well-aligned. + let bytes = <$t>::to_be_bytes(*self); + for _ in 0..alignment_bytes(bytes.len()) { + // Zero-pad + buffer.push_byte(0)?; + } + buffer.write(bytes.as_ref())?; + Ok(()) + } + } + + impl DeserializeCompact for $t { + fn decode_static(buffer: &mut I) -> Result { + let mut asset = [0u8; ::core::mem::size_of::<$t>()]; + buffer.skip(alignment_bytes(asset.len()))?; // Skip zero-padding + buffer.read(asset.as_mut())?; + Ok(<$t>::from_be_bytes(asset)) + } + } + }; +} + +impl_for_primitives!(u8, true); +impl_for_primitives!(u16, false); +impl_for_primitives!(u32, false); +impl_for_primitives!(usize, false); +impl_for_primitives!(u64, false); +impl_for_primitives!(u128, false); + +// Empty tuple `()`, i.e. the unit type takes up no space. +impl Serialize for () { + fn size_static(&self) -> usize { + 0 + } + + #[inline(always)] + fn size_dynamic(&self) -> usize { + 0 + } + + #[inline(always)] + fn encode_static(&self, _buffer: &mut O) -> Result<(), Error> { + Ok(()) + } +} + +impl Deserialize for () { + fn decode_static(_buffer: &mut I) -> Result { + Ok(()) + } +} + +/// To protect against malicious large inputs, vector size is limited when decoding. +pub const VEC_DECODE_LIMIT: usize = 100 * (1 << 20); // 100 MiB + +#[cfg(feature = "alloc")] +impl Serialize for Vec { + fn size_static(&self) -> usize { + 8 + } + + #[inline(always)] + fn size_dynamic(&self) -> usize { + if T::UNALIGNED_BYTES { + aligned_size(self.len()) + } else { + aligned_size(self.iter().map(|e| e.size()).sum()) + } + } + + #[inline(always)] + // Encode only the size of the vector. Elements will be encoded in the + // `encode_dynamic` method. + fn encode_static(&self, buffer: &mut O) -> Result<(), Error> { + if self.len() > VEC_DECODE_LIMIT { + return Err(Error::AllocationLimit) + } + let len: u64 = self.len().try_into().expect("msg.len() > u64::MAX"); + len.encode(buffer) + } + + fn encode_dynamic(&self, buffer: &mut O) -> Result<(), Error> { + // Bytes - Vec it a separate case without padding for each element. + // It should padded at the end if is not % ALIGN + if T::UNALIGNED_BYTES { + // SAFETY: `UNALIGNED_BYTES` only set for `u8`. + let bytes = unsafe { ::core::mem::transmute::<&Vec, &Vec>(self) }; + buffer.write(bytes.as_slice())?; + for _ in 0..alignment_bytes(self.len()) { + buffer.push_byte(0)?; + } + } else { + for e in self.iter() { + e.encode(buffer)?; + } + } + Ok(()) + } +} + +#[cfg(feature = "alloc")] +impl Deserialize for Vec { + // Decode only the capacity of the vector. Elements will be decoded in the + // `decode_dynamic` method. The capacity is needed for iteration there. + fn decode_static(buffer: &mut I) -> Result { + let cap = u64::decode(buffer)?; + let cap: usize = cap.try_into().map_err(|_| Error::AllocationLimit)?; + if cap > VEC_DECODE_LIMIT { + return Err(Error::AllocationLimit) + } + Ok(Vec::with_capacity(cap)) + } + + fn decode_dynamic(&mut self, buffer: &mut I) -> Result<(), Error> { + for _ in 0..self.capacity() { + // Bytes - Vec it a separate case without unpadding for each element. + // It should unpadded at the end if is not % ALIGN + if T::UNALIGNED_BYTES { + let byte = buffer.read_byte()?; + // SAFETY: `UNALIGNED_BYTES` implemented set for `u8`. + let _self = + unsafe { ::core::mem::transmute::<&mut Vec, &mut Vec>(self) }; + _self.push(byte); + } else { + self.push(T::decode(buffer)?); + } + } + + if T::UNALIGNED_BYTES { + buffer.skip(alignment_bytes(self.capacity()))?; + } + + Ok(()) + } +} + +impl Serialize for [T; N] { + fn size_static(&self) -> usize { + self.iter().map(|e| e.size_static()).sum() + } + + #[inline(always)] + fn size_dynamic(&self) -> usize { + self.iter().map(|e| e.size_dynamic()).sum() + } + + #[inline(always)] + fn encode_static(&self, buffer: &mut O) -> Result<(), Error> { + let bytes = unsafe { ::core::mem::transmute::<&[T; N], &[u8; N]>(self) }; + buffer.write(bytes.as_slice())?; + Ok(()) + } + + fn encode_dynamic(&self, buffer: &mut O) -> Result<(), Error> { + for e in self.iter() { + e.encode_dynamic(buffer)?; + } + + Ok(()) + } +} + +impl DeserializeCompact for [T; N] { + fn decode_static(buffer: &mut I) -> Result { + let mut bytes: [u8; N] = [0; N]; + buffer.read(bytes.as_mut())?; + let ref_typed: &[T; N] = unsafe { core::mem::transmute(&bytes) }; + let typed: [T; N] = unsafe { core::ptr::read(ref_typed) }; + Ok(typed) + } + + fn decode_dynamic(&mut self, buffer: &mut I) -> Result<(), Error> { + for e in self.iter_mut() { + e.decode_dynamic(buffer)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn validate(t: T) { + let bytes = t.to_bytes(); + let t2 = T::from_bytes(&bytes).expect("Roundtrip failed"); + assert_eq!(t, t2); + assert_eq!(t.to_bytes(), t2.to_bytes()); + + let mut vec = Vec::new(); + t.encode_static(&mut vec).expect("Encode failed"); + assert_eq!(vec.len(), t.size_static()); + } + + fn validate_enum(t: T) { + let bytes = t.to_bytes(); + let t2 = T::from_bytes(&bytes).expect("Roundtrip failed"); + assert_eq!(t, t2); + assert_eq!(t.to_bytes(), t2.to_bytes()); + + let mut vec = Vec::new(); + t.encode_static(&mut vec).expect("Encode failed"); + assert_eq!(vec.len(), t.size_static()); + t.encode_dynamic(&mut vec).expect("Encode failed"); + assert_eq!(vec.len(), t.size()); + + let mut vec2 = Vec::new(); + t.encode_dynamic(&mut vec2).expect("Encode failed"); + assert_eq!(vec2.len(), t.size_dynamic()); + } + + #[test] + fn test_compact_encode_decode() { + validate(()); + validate(123u8); + validate(u8::MAX); + validate(123u16); + validate(u16::MAX); + validate(123u32); + validate(u32::MAX); + validate(123u64); + validate(u64::MAX); + validate(123u128); + validate(u128::MAX); + validate(Vec::::new()); + validate(Vec::::new()); + validate(Vec::::new()); + validate(Vec::::new()); + validate(Vec::::new()); + validate(vec![1u8]); + validate(vec![1u16]); + validate(vec![1u32]); + validate(vec![1u64]); + validate(vec![1u128]); + validate(vec![1u8, 2u8]); + validate(vec![1u16, 2u16]); + validate(vec![1u32, 2u32]); + validate(vec![1u64, 2u64]); + validate(vec![1u128, 2u128]); + + #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] + struct TestStruct1 { + a: u8, + b: u16, + } + + let t = TestStruct1 { a: 123, b: 456 }; + assert_eq!(t.size_static(), 16); + assert_eq!(t.size(), 16); + validate(t); + + #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] + struct TestStruct2 { + a: u8, + v: Vec, + b: u16, + arr0: [u8; 0], + arr1: [u8; 2], + arr2: [u16; 3], + arr3: [u64; 4], + } + + validate(TestStruct2 { + a: 123, + v: vec![1, 2, 3], + b: 456, + arr0: [], + arr1: [1, 2], + arr2: [1, 2, u16::MAX], + arr3: [0, 3, 1111, u64::MAX], + }); + + #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] + #[repr(transparent)] + struct TestStruct3([u8; 64]); + + let t = TestStruct3([1; 64]); + assert_eq!(t.size_static(), 64); + assert_eq!(t.size(), 64); + validate(t); + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[canonical(prefix = 1u64)] + struct Prefixed1 { + a: [u8; 3], + b: Vec, + } + validate(Prefixed1 { + a: [1, 2, 3], + b: vec![4, 5, 6], + }); + + #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] + #[repr(u8)] + enum TestEnum1 { + A, + B, + C = 0x13, + D, + } + + validate(TestEnum1::A); + validate(TestEnum1::B); + validate(TestEnum1::C); + validate(TestEnum1::D); + + #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] + enum TestEnum2 { + A(u8), + B([u8; 3]), + C(Vec), + } + + validate_enum(TestEnum2::A(2)); + validate_enum(TestEnum2::B([1, 2, 3])); + validate_enum(TestEnum2::C(vec![1, 2, 3])); + + #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] + #[canonical(prefix = 2u64)] + struct Prefixed2(u16); + validate(Prefixed2(u16::MAX)); + + assert_eq!( + &Prefixed1 { + a: [1, 2, 3], + b: vec![4, 5] + } + .to_bytes()[..8], + &[0u8, 0, 0, 0, 0, 0, 0, 1] + ); + assert_eq!( + Prefixed2(u16::MAX).to_bytes(), + [0u8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0xff, 0xff] + ); + } +} From bf61b73274376fd66cd2e03ddd42e2fdee2d9cc7 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 1 Feb 2024 22:15:12 +0200 Subject: [PATCH 02/77] Migrate compact-derive from fuel-core --- Cargo.toml | 2 + fuel-compression/Cargo.toml | 31 ++ fuel-compression/README.md | 34 ++ fuel-compression/src/compaction.rs | 332 ++++++++++++++++ fuel-compression/src/lib.rs | 20 + .../src/registry/block_section.rs | 197 +++++++++ fuel-compression/src/registry/db.rs | 20 + fuel-compression/src/registry/in_memory.rs | 110 ++++++ fuel-compression/src/registry/key.rs | 176 +++++++++ fuel-compression/src/registry/mod.rs | 283 +++++++++++++ fuel-derive/Cargo.toml | 3 +- .../{attribute.rs => canonical_attribute.rs} | 0 fuel-derive/src/compact.rs | 374 ++++++++++++++++++ fuel-derive/src/deserialize.rs | 2 +- fuel-derive/src/lib.rs | 10 +- fuel-derive/src/serialize.rs | 2 +- fuel-tx/Cargo.toml | 3 +- fuel-tx/src/lib.rs | 3 + fuel-tx/src/transaction.rs | 1 + fuel-tx/src/transaction/policies.rs | 13 +- fuel-tx/src/transaction/types/create.rs | 2 + fuel-tx/src/transaction/types/input.rs | 27 +- fuel-tx/src/transaction/types/input/coin.rs | 22 +- .../src/transaction/types/input/contract.rs | 10 +- .../src/transaction/types/input/message.rs | 47 ++- fuel-tx/src/transaction/types/mint.rs | 8 +- fuel-tx/src/transaction/types/output.rs | 21 +- .../src/transaction/types/output/contract.rs | 6 +- fuel-tx/src/transaction/types/script.rs | 6 +- fuel-tx/src/transaction/types/storage.rs | 1 + fuel-tx/src/transaction/types/utxo_id.rs | 25 +- fuel-tx/src/transaction/types/witness.rs | 2 + fuel-tx/src/tx_pointer.rs | 1 + fuel-types/Cargo.toml | 3 +- fuel-types/src/array_types.rs | 4 +- fuel-types/src/numeric_types.rs | 1 + fuel-vm/Cargo.toml | 4 +- fuel-vm/src/lib.rs | 3 + 38 files changed, 1758 insertions(+), 51 deletions(-) create mode 100644 fuel-compression/Cargo.toml create mode 100644 fuel-compression/README.md create mode 100644 fuel-compression/src/compaction.rs create mode 100644 fuel-compression/src/lib.rs create mode 100644 fuel-compression/src/registry/block_section.rs create mode 100644 fuel-compression/src/registry/db.rs create mode 100644 fuel-compression/src/registry/in_memory.rs create mode 100644 fuel-compression/src/registry/key.rs create mode 100644 fuel-compression/src/registry/mod.rs rename fuel-derive/src/{attribute.rs => canonical_attribute.rs} (100%) create mode 100644 fuel-derive/src/compact.rs diff --git a/Cargo.toml b/Cargo.toml index 6386044719..0516fd5696 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "fuel-asm", + "fuel-compression", "fuel-crypto", "fuel-merkle", "fuel-storage", @@ -22,6 +23,7 @@ version = "0.44.0" [workspace.dependencies] fuel-asm = { version = "0.44.0", path = "fuel-asm", default-features = false } +fuel-compression = { version = "0.44.0", path = "fuel-compression", default-features = false } fuel-crypto = { version = "0.44.0", path = "fuel-crypto", default-features = false } fuel-derive = { version = "0.44.0", path = "fuel-derive", default-features = false } fuel-merkle = { version = "0.44.0", path = "fuel-merkle", default-features = false } diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml new file mode 100644 index 0000000000..4bab089d9d --- /dev/null +++ b/fuel-compression/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "fuel-compression" +version = { workspace = true } +authors = { workspace = true } +categories = ["cryptography::cryptocurrencies"] +edition = { workspace = true } +homepage = { workspace = true } +keywords = ["blockchain", "cryptocurrencies", "fuel-compression"] +license = { workspace = true } +repository = { workspace = true } +description = "Compression and decompression of Fuel blocks for DA storage." + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde-big-array = "0.5" + +postcard = { version = "1.0", features = ["use-std"] } +bincode = "1.3" + +paste = "1.0" + +fuel-derive = { workspace = true } + +[dev-dependencies] +fuel-compression = { path = "." } # Self-dependency needed by test for macros +fuel-asm = { workspace = true } +fuel-types = { workspace = true, features = ["serde"] } +fuel-tx = { workspace = true } + +[features] +test-helpers = [] diff --git a/fuel-compression/README.md b/fuel-compression/README.md new file mode 100644 index 0000000000..23767f0ea1 --- /dev/null +++ b/fuel-compression/README.md @@ -0,0 +1,34 @@ +# Compression and decompression of fuel-types for the DA layer + +## Compressed block header + +Each compressed block begins with a single-byte version field, so that it's possible to change the format later. + +## Temporal registry + +This crate provides offchain registries for different types such as `AssetId`, `ContractId`, scripts, and predicates. Each registry is a key-value store with three-byte key. The registires are essentially compression caches. The three byte key allows cache size of 16 million values before reregistering the older values. + +The registries allow replacing repeated objects with their respective keys, so if an object +is used multiple times in a short interval (couple of months, maybe), then the full value +exists on only a single uncompressed block, + +### Fraud proofs + +Compressed block will start with 32 bytes of merkle root over all compression smts, followed by newly registered values along with their keys. Using an SMT provides flexibility around the algorithm we use to define keys without knowing how exactly values were chosen to be registered. + +Each registry also uses an SMT. Since the keys are three bytes long, the depth of the SMT is capped at 24 levels. + + + - More efficient for fraud proofs instead of needing to provide entire previous blocks with proofs + +## Compression of `UtxoIds` + +Since each `UtxoId` only appears once, there's no point in registering them. Instead, they are replaced with `TxPointer`s (7 bytes worst case), which are still unique. + +### Fraud proofs + +During fraud proofs we need to use the `prev_root` to prove that the referenced block height is part of the chain. + +## Other techniques + +- These techniques should be good enough for now, but there are lots of other interesting ideas for this. diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs new file mode 100644 index 0000000000..29433f303e --- /dev/null +++ b/fuel-compression/src/compaction.rs @@ -0,0 +1,332 @@ +use std::marker::PhantomData; + +use serde::{ + Deserialize, + Serialize, +}; + +use crate::{ + registry::{ + access::{ + self, + *, + }, + add_keys, + block_section::WriteTo, + next_keys, + ChangesPerTable, + CountPerTable, + KeyPerTable, + RegistryDb, + Table, + }, + Key, +}; + +#[must_use] +pub struct CompactionContext<'a, R> { + /// The registry + reg: &'a mut R, + /// These are the keys where writing started + start_keys: KeyPerTable, + /// The next keys to use for each table + next_keys: KeyPerTable, + /// Keys in range next_keys..safe_keys_start + /// could be overwritten by the compaction, + /// and cannot be used for new values. + safe_keys_start: KeyPerTable, + changes: ChangesPerTable, +} +impl<'a, R: RegistryDb> CompactionContext<'a, R> { + /// Run the compaction for the given target, returning the compacted data. + /// Changes are applied to the registry, and then returned as well. + pub fn run( + reg: &'a mut R, + target: C, + ) -> (C::Compact, ChangesPerTable) { + let start_keys = next_keys(reg); + let next_keys = start_keys; + let key_limits = target.count(); + let safe_keys_start = add_keys(next_keys, key_limits); + + let mut ctx = Self { + reg, + start_keys, + next_keys, + safe_keys_start, + changes: ChangesPerTable::from_start_keys(start_keys), + }; + + let compacted = target.compact(&mut ctx); + ctx.changes.apply_to_registry(ctx.reg); + (compacted, ctx.changes) + } +} + +impl<'a, R: RegistryDb> CompactionContext<'a, R> { + /// Convert a value to a key + /// If necessary, store the value in the changeset and allocate a new key. + pub fn to_key(&mut self, value: T::Type) -> Key + where + KeyPerTable: access::AccessCopy>, + KeyPerTable: access::AccessMut>, + ChangesPerTable: + access::AccessRef> + access::AccessMut>, + { + // Check if the value is within the current changeset + if let Some(key) = + >>::get(&self.changes) + .lookup_value(&value) + { + return key; + } + + // Check if the registry contains this value already + if let Some(key) = self.reg.index_lookup::(&value) { + let start: Key = self.start_keys.value(); + let end: Key = self.safe_keys_start.value(); + // Check if the value is in the possibly-overwritable range + if !key.is_between(start, end) { + return key; + } + } + // Allocate a new key for this + let key = >>::get_mut(&mut self.next_keys) + .take_next(); + >>::get_mut(&mut self.changes) + .values + .push(value); + key + } +} + +/// Convert data to reference-based format +pub trait Compactable { + type Compact: Clone + Serialize + for<'a> Deserialize<'a>; + + /// Count max number of each key type, for upper limit of overwritten keys + fn count(&self) -> CountPerTable; + + fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact; + + fn decompact(compact: Self::Compact, reg: &R) -> Self; +} + +macro_rules! identity_compaction { + ($t:ty) => { + impl Compactable for $t { + type Compact = Self; + + fn count(&self) -> CountPerTable { + CountPerTable::default() + } + + fn compact( + &self, + _ctx: &mut CompactionContext, + ) -> Self::Compact { + *self + } + + fn decompact(compact: Self::Compact, _reg: &R) -> Self { + compact + } + } + }; +} + +identity_compaction!(u8); +identity_compaction!(u16); +identity_compaction!(u32); +identity_compaction!(u64); +identity_compaction!(u128); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArrayWrapper Deserialize<'a>>( + #[serde(with = "serde_big_array::BigArray")] pub [T; S], +); + +impl Compactable for [T; S] +where + T: Compactable + Clone + Serialize + for<'a> Deserialize<'a>, +{ + type Compact = ArrayWrapper; + + fn count(&self) -> CountPerTable { + let mut count = CountPerTable::default(); + for item in self.iter() { + count += item.count(); + } + count + } + + fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact { + ArrayWrapper(self.clone().map(|item| item.compact(ctx))) + } + + fn decompact(compact: Self::Compact, reg: &R) -> Self { + compact.0.map(|item| T::decompact(item, reg)) + } +} + +impl Compactable for Vec +where + T: Compactable + Clone + Serialize + for<'a> Deserialize<'a>, +{ + type Compact = Vec; + + fn count(&self) -> CountPerTable { + let mut count = CountPerTable::default(); + for item in self.iter() { + count += item.count(); + } + count + } + + fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact { + self.iter().map(|item| item.compact(ctx)).collect() + } + + fn decompact(compact: Self::Compact, reg: &R) -> Self { + compact + .into_iter() + .map(|item| T::decompact(item, reg)) + .collect() + } +} + +impl Compactable for PhantomData { + type Compact = (); + + fn count(&self) -> CountPerTable { + CountPerTable::default() + } + + fn compact(&self, _ctx: &mut CompactionContext) -> Self::Compact { + () + } + + fn decompact(_compact: Self::Compact, _reg: &R) -> Self { + Self + } +} + +#[cfg(test)] +mod tests { + use crate::{ + registry::{ + in_memory::InMemoryRegistry, + tables, + CountPerTable, + }, + Key, + RegistryDb, + }; + use fuel_compression::Compactable as _; // Hack for derive + use fuel_derive::Compact; + use fuel_types::{ + Address, + AssetId, + }; + use serde::{ + Deserialize, + Serialize, + }; + + use super::{ + Compactable, + CompactionContext, + }; + + #[derive(Debug, Clone, PartialEq)] + struct ManualExample { + a: Address, + b: Address, + c: u64, + } + + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + struct ManualExampleCompact { + a: Key, + b: Key, + c: u64, + } + + impl Compactable for ManualExample { + type Compact = ManualExampleCompact; + + fn count(&self) -> crate::registry::CountPerTable { + CountPerTable { + Address: 2, + ..Default::default() + } + } + + fn compact( + &self, + ctx: &mut CompactionContext, + ) -> Self::Compact { + let a = ctx.to_key::(*self.a); + let b = ctx.to_key::(*self.b); + ManualExampleCompact { a, b, c: self.c } + } + + fn decompact(compact: Self::Compact, reg: &R) -> Self { + let a = Address::from(reg.read::(compact.a)); + let b = Address::from(reg.read::(compact.b)); + Self { a, b, c: compact.c } + } + } + + #[derive(Debug, Clone, PartialEq, Compact)] + struct AutomaticExample { + #[da_compress(registry = "AssetId")] + a: AssetId, + #[da_compress(registry = "AssetId")] + b: AssetId, + c: u32, + } + + #[test] + fn test_compaction_properties() { + let a = ManualExample { + a: Address::from([1u8; 32]), + b: Address::from([2u8; 32]), + c: 3, + }; + assert_eq!(a.count().Address, 2); + assert_eq!(a.count().AssetId, 0); + + let b = AutomaticExample { + a: AssetId::from([1u8; 32]), + b: AssetId::from([2u8; 32]), + c: 3, + }; + assert_eq!(b.count().Address, 0); + assert_eq!(b.count().AssetId, 2); + } + + #[test] + fn test_compaction_roundtrip() { + let target = ManualExample { + a: Address::from([1u8; 32]), + b: Address::from([2u8; 32]), + c: 3, + }; + let mut registry = InMemoryRegistry::default(); + let (compacted, _) = CompactionContext::run(&mut registry, target.clone()); + let decompacted = ManualExample::decompact(compacted, ®istry); + assert_eq!(target, decompacted); + + let target = AutomaticExample { + a: AssetId::from([1u8; 32]), + b: AssetId::from([2u8; 32]), + c: 3, + }; + let mut registry = fuel_compression::InMemoryRegistry::default(); + let (compacted, _) = + fuel_compression::CompactionContext::run(&mut registry, target.clone()); + let decompacted = AutomaticExample::decompact(compacted, ®istry); + assert_eq!(target, decompacted); + } +} diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs new file mode 100644 index 0000000000..25967d6295 --- /dev/null +++ b/fuel-compression/src/lib.rs @@ -0,0 +1,20 @@ +mod compaction; +mod registry; + +pub use compaction::{ + Compactable, + CompactionContext, +}; +pub use registry::{ + tables, + ChangesPerTable, + CountPerTable, + Key, + RegistryDb, + Table, +}; + +#[cfg(feature = "test-helpers")] +pub use registry::in_memory::InMemoryRegistry; + +pub use fuel_derive::Compact; diff --git a/fuel-compression/src/registry/block_section.rs b/fuel-compression/src/registry/block_section.rs new file mode 100644 index 0000000000..2e4e73a625 --- /dev/null +++ b/fuel-compression/src/registry/block_section.rs @@ -0,0 +1,197 @@ +use core::fmt; + +use serde::{ + ser::SerializeTuple, + Deserialize, + Serialize, +}; + +use super::{ + key::Key, + ChangesPerTable, + Table, +}; + +/// New registrations written to a specific table. +#[derive(Clone, PartialEq, Eq)] +pub struct WriteTo { + /// The values are inserted starting from this key + pub start_key: Key, + /// Values. inserted using incrementing ids starting from `start_key` + pub values: Vec, +} + +impl fmt::Debug for WriteTo +where + T::Type: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.values.is_empty() { + return f.write_str("WriteTo::EMPTY"); + } + + f.debug_struct("WriteTo") + .field("start_key", &self.start_key) + .field("values", &self.values) + .finish() + } +} + +impl WriteTo +where + T::Type: PartialEq, +{ + /// Reverse lookup. + /// TODO: possibly add a lookup table for this, if deemed necessary + pub fn lookup_value(&self, needle: &T::Type) -> Option> { + if *needle == T::Type::default() { + return Some(Key::DEFAULT_VALUE); + } + + let mut key = self.start_key; + for v in &self.values { + if v == needle { + return Some(key); + } + key = key.next(); + } + None + } +} + +/// Custom serialization is used to omit the start_key when the sequence is empty +impl Serialize for WriteTo +where + T: Table + Serialize, +{ + fn serialize(&self, serializer: S) -> Result { + let mut tup = serializer.serialize_tuple(2)?; + tup.serialize_element(&self.values)?; + if self.values.is_empty() { + tup.serialize_element(&())?; + } else { + tup.serialize_element(&self.start_key)?; + } + tup.end() + } +} + +impl<'de, T: Table> Deserialize<'de> for WriteTo +where + T: Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_tuple( + 2, + Self { + start_key: Key::ZERO, + values: Vec::new(), + }, + ) + } +} + +impl<'de, T: Table + Deserialize<'de>> serde::de::Visitor<'de> for WriteTo { + type Value = WriteTo; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(concat!("WriteTo<", stringify!(T), "> instance")) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let values: Vec = seq.next_element()?.ok_or( + serde::de::Error::invalid_length(0, &"WriteTo<_> with 2 elements"), + )?; + + if values.is_empty() { + let _: () = seq.next_element()?.ok_or(serde::de::Error::invalid_length( + 1, + &"WriteTo<_> with 2 elements", + ))?; + Ok(WriteTo { + start_key: Key::ZERO, + values, + }) + } else { + let start_key: Key = seq.next_element()?.ok_or( + serde::de::Error::invalid_length(1, &"WriteTo<_> with 2 elements"), + )?; + Ok(WriteTo { start_key, values }) + } + } +} + +/// Registeration section of the compressed block +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Registrations { + /// Merkle root of the registeration table merkle roots + pub tables_root: [u8; 32], + /// Changes per table + pub changes: ChangesPerTable, +} + +#[cfg(test)] +mod tests { + use super::*; + use bincode::Options; + use fuel_asm::op; + use fuel_tx::AssetId; + use fuel_types::Address; + + #[test] + fn test_tables() { + let original = Registrations { + tables_root: Default::default(), + changes: ChangesPerTable { + AssetId: WriteTo { + start_key: Key::try_from(100).unwrap(), + values: vec![*AssetId::from([0xa0; 32]), *AssetId::from([0xa1; 32])], + }, + Address: WriteTo { + start_key: Key::ZERO, + values: vec![*Address::from([0xc0; 32])], + }, + ScriptCode: WriteTo { + start_key: Key::ZERO, + values: vec![ + vec![op::addi(0x20, 0x20, 1), op::ret(0)] + .into_iter() + .collect(), + vec![op::muli(0x20, 0x20, 5), op::ret(1)] + .into_iter() + .collect(), + ], + }, + Witness: WriteTo { + start_key: Key::ZERO, + values: vec![], + }, + }, + }; + + let pc_compressed = postcard::to_stdvec(&original).unwrap(); + let pc_decompressed: Registrations = + postcard::from_bytes(&pc_compressed).unwrap(); + assert_eq!(original, pc_decompressed); + + let bc_opt = bincode::DefaultOptions::new().with_varint_encoding(); + + let bc_compressed = bc_opt.serialize(&original).unwrap(); + let bc_decompressed: Registrations = bc_opt.deserialize(&bc_compressed).unwrap(); + assert_eq!(original, bc_decompressed); + + println!("data: {original:?}"); + println!("postcard compressed size {}", pc_compressed.len()); + println!("bincode compressed size {}", bc_compressed.len()); + println!("postcard compressed: {:x?}", pc_compressed); + println!("bincode compressed: {:x?}", bc_compressed); + + // panic!("ok, just showing the results"); + } +} diff --git a/fuel-compression/src/registry/db.rs b/fuel-compression/src/registry/db.rs new file mode 100644 index 0000000000..40c729c31e --- /dev/null +++ b/fuel-compression/src/registry/db.rs @@ -0,0 +1,20 @@ +use super::{ + Key, + Table, +}; + +pub trait RegistryDb { + /// Get next key for the given table. This is where the next write should start at. + /// The result of this function is just a suggestion, and the caller may choose to + /// ignore it, although it's rare that they would know better. + fn next_key(&self) -> Key; + + /// Read a value from the registry by key + fn read(&self, key: Key) -> T::Type; + + /// Write a continuous sequence of values to the registry + fn batch_write(&mut self, start_key: Key, values: Vec); + + /// Lookup a key by value + fn index_lookup(&self, value: &T::Type) -> Option>; +} diff --git a/fuel-compression/src/registry/in_memory.rs b/fuel-compression/src/registry/in_memory.rs new file mode 100644 index 0000000000..f55b001597 --- /dev/null +++ b/fuel-compression/src/registry/in_memory.rs @@ -0,0 +1,110 @@ +use std::collections::HashMap; + +use super::{ + db::*, + key::RawKey, + Key, + Table, +}; + +/// Simple and inefficient in-memory registry for testing purposes. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct InMemoryRegistry { + next_keys: HashMap<&'static str, RawKey>, + storage: HashMap<&'static str, HashMap>>, + index: HashMap<&'static str, HashMap, RawKey>>, +} + +impl RegistryDb for InMemoryRegistry { + fn next_key(&self) -> Key { + Key::from_raw(self.next_keys.get(T::NAME).copied().unwrap_or(RawKey::ZERO)) + } + + fn read(&self, key: Key) -> T::Type { + if key == Key::DEFAULT_VALUE { + return T::Type::default(); + } + + self.storage + .get(T::NAME) + .and_then(|table| table.get(&key.raw())) + .map(|bytes| postcard::from_bytes(bytes).expect("Invalid value in registry")) + .unwrap_or_default() + } + + fn batch_write(&mut self, start_key: Key, values: Vec) { + let empty = values.is_empty(); + if !empty && start_key == Key::DEFAULT_VALUE { + panic!("Cannot write to the default value key"); + } + let table = self.storage.entry(T::NAME).or_default(); + let mut key = start_key.raw(); + for value in values.into_iter() { + let value = postcard::to_stdvec(&value).unwrap(); + let mut prefix = value.clone(); + prefix.truncate(32); + self.index.entry(T::NAME).or_default().insert(prefix, key); + table.insert(key, value); + key = key.next(); + } + if !empty { + self.next_keys.insert(T::NAME, key); + } + } + + fn index_lookup(&self, value: &T::Type) -> Option> { + if *value == T::Type::default() { + return Some(Key::DEFAULT_VALUE); + } + + let needle = postcard::to_stdvec(value).unwrap(); + let mut prefix = needle.clone(); + prefix.truncate(32); + if let Some(cand) = self.index.get(T::NAME)?.get(&prefix).copied() { + let cand_val = self.storage.get(T::NAME)?.get(&cand)?; + if *cand_val == needle { + return Some(Key::from_raw(cand)); + } + } + + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + tables, + Key, + }; + + #[test] + fn in_memory_registry_works() { + let mut reg = InMemoryRegistry::default(); + + // Empty + assert_eq!( + reg.read(Key::::try_from(100).unwrap()), + [0; 32] + ); + + // Write + reg.batch_write( + Key::::from_raw(RawKey::try_from(100u32).unwrap()), + vec![[1; 32], [2; 32]], + ); + + // Read + assert_eq!( + reg.read(Key::::try_from(100).unwrap()), + [1; 32] + ); + + // Index + assert_eq!( + reg.index_lookup(&[1; 32]), + Some(Key::::try_from(100).unwrap()) + ); + } +} diff --git a/fuel-compression/src/registry/key.rs b/fuel-compression/src/registry/key.rs new file mode 100644 index 0000000000..0b4379ab69 --- /dev/null +++ b/fuel-compression/src/registry/key.rs @@ -0,0 +1,176 @@ +use core::fmt; +use std::marker::PhantomData; + +use serde::{ + Deserialize, + Serialize, +}; + +use super::Table; + +/// Untyped key pointing to a registry table entry. +/// The last key (all bits set) is reserved for the default value and cannot be written +/// to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct RawKey([u8; Self::SIZE]); +impl RawKey { + pub const DEFAULT_VALUE: Self = Self([u8::MAX; Self::SIZE]); + pub const MAX_WRITABLE: Self = Self([u8::MAX, u8::MAX, u8::MAX - 1]); + pub const SIZE: usize = 3; + pub const ZERO: Self = Self([0; Self::SIZE]); + + pub fn as_u32(self) -> u32 { + u32::from_be_bytes([0, self.0[0], self.0[1], self.0[2]]) + } + + /// Wraps around just below max/default value. + pub fn add_u32(self, rhs: u32) -> Self { + let lhs = self.as_u32() as u64; + let rhs = rhs as u64; + // Safety: cannot overflow as both operands are limited to 32 bits + let result = (lhs + rhs) % (Self::DEFAULT_VALUE.as_u32() as u64); + // Safety: cannot truncate as we are already limited to 24 bits by modulo + let v = result as u32; + let v = v.to_be_bytes(); + Self([v[1], v[2], v[3]]) + } + + /// Wraps around just below max/default value. + pub fn next(self) -> Self { + self.add_u32(1) + } + + /// Is `self` between `start` and `end`? i.e. in the half-open logical range + /// `start`..`end`, so that wrap-around cases are handled correctly. + /// + /// Panics if max/default value is used. + pub fn is_between(self, start: Self, end: Self) -> bool { + assert!( + self != Self::DEFAULT_VALUE, + "Cannot use max/default value in is_between" + ); + assert!( + start != Self::DEFAULT_VALUE, + "Cannot use max/default value in is_between" + ); + assert!( + end != Self::DEFAULT_VALUE, + "Cannot use max/default value in is_between" + ); + + let low = start.as_u32(); + let high = end.as_u32(); + let v = self.as_u32(); + + if high >= low { + low <= v && v < high + } else { + v < high || v >= low + } + } +} +impl TryFrom for RawKey { + type Error = &'static str; + + fn try_from(value: u32) -> Result { + let v = value.to_be_bytes(); + if v[0] != 0 { + return Err("RawKey must be less than 2^24"); + } + + let mut bytes = [0u8; 3]; + bytes.copy_from_slice(&v[1..]); + Ok(Self(bytes)) + } +} + +/// Typed key to a registry table entry. +/// The last key (all bits set) is reserved for the default value and cannot be written +/// to. +#[derive(Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Key(RawKey, PhantomData); +impl Clone for Key { + fn clone(&self) -> Self { + Self(self.0, PhantomData) + } +} +impl Copy for Key {} + +impl PartialEq> for Key { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl Key { + /// This key is reserved for the default value and cannot be written to. + pub const DEFAULT_VALUE: Self = Self(RawKey::DEFAULT_VALUE, PhantomData); + /// This is the first writable key. + pub const ZERO: Self = Self(RawKey::ZERO, PhantomData); + + pub fn raw(&self) -> RawKey { + self.0 + } + + pub fn from_raw(raw: RawKey) -> Self { + Self(raw, PhantomData) + } + + /// Wraps around at limit, i.e. one below the max/default value + pub fn add_u32(self, rhs: u32) -> Self { + Self(self.0.add_u32(rhs), PhantomData) + } + + /// Wraps around at limit, i.e. one below the max/default value + pub fn next(self) -> Self { + Self(self.0.next(), PhantomData) + } + + /// Is `self` between `start` and `end`? i.e. in the half-open logical range + /// `start`..`end`, so that wrap-around cases are handled correctly. + pub fn is_between(self, start: Self, end: Self) -> bool { + self.0.is_between(start.0, end.0) + } + + /// Increments the key by one, and returns the previous value. + /// Skips the max/default value. + pub fn take_next(&mut self) -> Self { + let result = *self; + self.0 = self.0.next(); + result + } +} + +impl TryFrom for Key { + type Error = &'static str; + + fn try_from(value: u32) -> Result { + Ok(Self(RawKey::try_from(value)?, PhantomData)) + } +} + +impl fmt::Debug for Key { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if *self == Self::DEFAULT_VALUE { + write!(f, "Key<{}>::DEFAULT_VALUE", T::NAME) + } else { + write!(f, "Key<{}>({})", T::NAME, self.0.as_u32()) + } + } +} + +#[cfg(test)] +mod tests { + use super::RawKey; + + #[test] + fn key_next() { + assert_eq!(RawKey::ZERO.next(), RawKey([0, 0, 1])); + assert_eq!(RawKey::ZERO.next().next(), RawKey([0, 0, 2])); + assert_eq!(RawKey([0, 0, 255]).next(), RawKey([0, 1, 0])); + assert_eq!(RawKey([0, 1, 255]).next(), RawKey([0, 2, 0])); + assert_eq!(RawKey([0, 255, 255]).next(), RawKey([1, 0, 0])); + assert_eq!(RawKey::MAX_WRITABLE.next(), RawKey::ZERO); + } +} diff --git a/fuel-compression/src/registry/mod.rs b/fuel-compression/src/registry/mod.rs new file mode 100644 index 0000000000..7b9486dd40 --- /dev/null +++ b/fuel-compression/src/registry/mod.rs @@ -0,0 +1,283 @@ +use serde::{ + Deserialize, + Serialize, +}; + +pub(crate) mod block_section; +pub mod db; +pub(crate) mod in_memory; +mod key; + +use self::block_section::WriteTo; +pub use self::{ + db::RegistryDb, + key::Key, +}; + +mod _private { + pub trait Seal {} +} + +pub trait Table: _private::Seal { + const NAME: &'static str; + type Type: PartialEq + Default + Serialize + for<'de> Deserialize<'de>; +} + +pub mod access { + pub trait AccessCopy { + fn value(&self) -> V; + } + + pub trait AccessRef { + fn get(&self) -> &V; + } + + pub trait AccessMut { + fn get_mut(&mut self) -> &mut V; + } +} + +macro_rules! tables { + // $index muse use increasing numbers starting from zero + ($($name:ident: $ty:ty),*$(,)?) => { + pub mod tables { + $( + /// Specifies the table to use for a given key. + /// The data is separated to tables based on the data type being stored. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] + pub struct $name; + + impl super::_private::Seal for $name {} + impl super::Table for $name { + const NAME: &'static str = stringify!($name); + type Type = $ty; + } + )* + } + + /// One counter per table + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] + #[allow(non_snake_case)] // The field names match table type names eactly + #[non_exhaustive] + pub struct CountPerTable { + $(pub $name: usize),* + } + + impl CountPerTable {$( + /// Custom constructor per table + #[allow(non_snake_case)] // The field names match table type names eactly + pub fn $name(value: usize) -> Self { + Self { + $name: value, + ..Self::default() + } + } + )*} + + $( + impl access::AccessCopy for CountPerTable { + fn value(&self) -> usize { + self.$name + } + } + )* + + impl core::ops::Add for CountPerTable { + type Output = Self; + + fn add(self, rhs: CountPerTable) -> Self::Output { + Self { + $($name: self.$name + rhs.$name),* + } + } + } + + impl core::ops::AddAssign for CountPerTable { + fn add_assign(&mut self, rhs: CountPerTable) { + $(self.$name += rhs.$name);* + } + } + + /// One key value per table + #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] + #[allow(non_snake_case)] // The field names match table type names eactly + #[non_exhaustive] + pub struct KeyPerTable { + $(pub $name: Key),* + } + + impl Default for KeyPerTable { + fn default() -> Self { + Self { + $($name: Key::ZERO,)* + } + } + } + + $( + impl access::AccessCopy> for KeyPerTable { + fn value(&self) -> Key { + self.$name + } + } + impl access::AccessRef> for KeyPerTable { + fn get(&self) -> &Key { + &self.$name + } + } + impl access::AccessMut> for KeyPerTable { + fn get_mut(&mut self) -> &mut Key { + &mut self.$name + } + } + )* + + pub fn next_keys(reg: &mut R) -> KeyPerTable { + KeyPerTable { + $( $name: reg.next_key(), )* + } + } + + /// Used to add together keys and counts to deterimine possible overwrite range + pub fn add_keys(keys: KeyPerTable, counts: CountPerTable) -> KeyPerTable { + KeyPerTable { + $( + $name: keys.$name.add_u32(counts.$name.try_into() + .expect("Count too large. Shoudn't happen as we control inputs here.") + ), + )* + } + } + + /// Registeration changes per table + #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] + #[allow(non_snake_case)] // The field names match table type names eactly + #[non_exhaustive] + pub struct ChangesPerTable { + $(pub $name: WriteTo),* + } + + impl ChangesPerTable { + pub fn is_empty(&self) -> bool { + true $(&& self.$name.values.is_empty())* + } + + pub fn from_start_keys(start_keys: KeyPerTable) -> Self { + Self { + $($name: WriteTo { + start_key: start_keys.$name, + values: Vec::new(), + }),* + } + } + + /// Apply changes to the registry db + pub fn apply_to_registry(&self, reg: &mut R) { + $( + reg.batch_write(self.$name.start_key, self.$name.values.clone()); + )* + } + } + + $( + impl access::AccessRef> for ChangesPerTable { + fn get(&self) -> &WriteTo { + &self.$name + } + } + impl access::AccessMut> for ChangesPerTable { + fn get_mut(&mut self) -> &mut WriteTo { + &mut self.$name + } + } + )* + }; +} + +tables!( + AssetId: [u8; 32], + Address: [u8; 32], + ScriptCode: Vec, + Witness: Vec, +); + +#[cfg(test)] +mod tests { + use fuel_types::AssetId; + use tests::key::RawKey; + + use super::*; + + #[test] + fn test_in_memory_db() { + let mut reg = in_memory::InMemoryRegistry::default(); + + // Empty + assert_eq!( + reg.read(Key::::try_from(100).unwrap()), + [0; 32] + ); + assert_eq!( + reg.index_lookup(&*AssetId::from([1; 32])), + None::> + ); + + // Write + reg.batch_write( + Key::::from_raw(RawKey::try_from(100u32).unwrap()), + vec![[1; 32], [2; 32]], + ); + assert_eq!( + reg.read(Key::::try_from(100).unwrap()), + [1; 32] + ); + assert_eq!( + reg.read(Key::::try_from(101).unwrap()), + [2; 32] + ); + assert_eq!( + reg.read(Key::::try_from(102).unwrap()), + [0; 32] + ); + + // Overwrite + reg.batch_write( + Key::::from_raw(RawKey::try_from(99u32).unwrap()), + vec![[10; 32], [11; 32]], + ); + assert_eq!( + reg.read(Key::::try_from(99).unwrap()), + [10; 32] + ); + assert_eq!( + reg.read(Key::::try_from(100).unwrap()), + [11; 32] + ); + + // Wrapping + reg.batch_write( + Key::::from_raw(RawKey::MAX_WRITABLE), + vec![[3; 32], [4; 32]], + ); + + assert_eq!( + reg.read(Key::::from_raw(RawKey::MAX_WRITABLE)), + [3; 32] + ); + + assert_eq!( + reg.read(Key::::from_raw(RawKey::ZERO)), + [4; 32] + ); + + assert_eq!( + reg.index_lookup(&*AssetId::from([3; 32])), + Some(Key::::from_raw(RawKey::MAX_WRITABLE)) + ); + + assert_eq!( + reg.index_lookup(&*AssetId::from([4; 32])), + Some(Key::::from_raw(RawKey::ZERO)) + ); + } +} diff --git a/fuel-derive/Cargo.toml b/fuel-derive/Cargo.toml index 6bddd0d857..17643dadab 100644 --- a/fuel-derive/Cargo.toml +++ b/fuel-derive/Cargo.toml @@ -8,7 +8,7 @@ homepage = { workspace = true } keywords = ["blockchain", "cryptocurrencies", "fuel-vm", "vm"] license = { workspace = true } repository = { workspace = true } -description = "FuelVM (de)serialization derive macros for `fuel-vm` data structures." +description = "FuelVM (de)serialization and compaction derive macros for `fuel-vm` data structures." [lib] proc-macro = true @@ -18,3 +18,4 @@ quote = "1" syn = { version = "2", features = ["full"] } proc-macro2 = "1" synstructure = "0.13" +regex = "1" diff --git a/fuel-derive/src/attribute.rs b/fuel-derive/src/canonical_attribute.rs similarity index 100% rename from fuel-derive/src/attribute.rs rename to fuel-derive/src/canonical_attribute.rs diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compact.rs new file mode 100644 index 0000000000..f342df29af --- /dev/null +++ b/fuel-derive/src/compact.rs @@ -0,0 +1,374 @@ +use proc_macro2::TokenStream as TokenStream2; +use quote::{ + format_ident, + quote, +}; + +use regex::Regex; + +const ATTR: &str = "da_compress"; + +/// Field attributes +pub enum FieldAttrs { + /// Skipped when compacting, and must be reconstructed when decompacting. + Skip, + /// Compacted recursively. + Normal, + /// This value is compacted into a registry lookup. + Registry(String), +} +impl FieldAttrs { + pub fn parse(attrs: &[syn::Attribute]) -> Self { + let re_registry = Regex::new(r#"^registry\s*=\s*"([a-zA-Z_]+)"$"#).unwrap(); + + let mut result = Self::Normal; + for attr in attrs { + if attr.style != syn::AttrStyle::Outer { + continue; + } + + if let syn::Meta::List(ml) = &attr.meta { + if ml.path.segments.len() == 1 && ml.path.segments[0].ident == ATTR { + if !matches!(result, Self::Normal) { + panic!("Duplicate attribute: {}", ml.tokens); + } + + let attr_contents = ml.tokens.to_string(); + if attr_contents == "skip" { + result = Self::Skip; + } else if let Some(m) = re_registry.captures(&attr_contents) { + result = Self::Registry(m.get(1).unwrap().as_str().to_owned()); + } else { + panic!("Invalid attribute: {}", ml.tokens); + } + } + } + } + + result + } +} + +/// Map field definitions to compacted field definitions. +fn field_defs(fields: &syn::Fields) -> TokenStream2 { + let mut defs = TokenStream2::new(); + + for field in fields { + let attrs = FieldAttrs::parse(&field.attrs); + defs.extend(match &attrs { + FieldAttrs::Skip => quote! {}, + FieldAttrs::Normal => { + let ty = &field.ty; + let cty = quote! { + <#ty as ::fuel_compression::Compactable>::Compact + }; + if let Some(fname) = field.ident.as_ref() { + quote! { #fname: #cty, } + } else { + quote! { #cty, } + } + } + FieldAttrs::Registry(registry) => { + let reg_ident = format_ident!("{}", registry); + let cty = quote! { + ::fuel_compression::Key<::fuel_compression::tables::#reg_ident> + }; + if let Some(fname) = field.ident.as_ref() { + quote! { #fname: #cty, } + } else { + quote! { #cty, } + } + } + }); + } + + match fields { + syn::Fields::Named(_) => quote! {{ #defs }}, + syn::Fields::Unnamed(_) => quote! {(#defs)}, + syn::Fields::Unit => quote! {}, + } +} + +/// Construct compact version of the struct from the original one +fn construct_compact( + // The structure to construct, i.e. struct name or enum variant path + compact: &TokenStream2, + variant: &synstructure::VariantInfo<'_>, +) -> TokenStream2 { + let bound_fields: TokenStream2 = variant + .bindings() + .iter() + .map(|binding| { + let attrs = FieldAttrs::parse(&binding.ast().attrs); + let ty = &binding.ast().ty; + let cname = format_ident!("{}_c", binding.binding); + + match attrs { + FieldAttrs::Skip => quote! {}, + FieldAttrs::Normal => { + quote! { + let #cname = <#ty as Compactable>::compact(&#binding, ctx); + } + } + FieldAttrs::Registry(registry) => { + let reg_ident = format_ident!("{}", registry); + let cty = quote! { + Key< + tables::#reg_ident + > + }; + quote! { + let #cname: #cty = ctx.to_key( + ::Type::from(#binding.clone()) + ); + } + } + } + }) + .collect(); + + let construct_fields: TokenStream2 = variant + .bindings() + .iter() + .map(|binding| { + let attrs = FieldAttrs::parse(&binding.ast().attrs); + if matches!(attrs, FieldAttrs::Skip) { + return quote! {}; + } + let cname = format_ident!("{}_c", binding.binding); + if let Some(fname) = &binding.ast().ident { + quote! { #fname: #cname, } + } else { + quote! { #cname, } + } + }) + .collect(); + + let construct_fields = match variant.ast().fields { + syn::Fields::Named(_) => quote! {{ #construct_fields }}, + syn::Fields::Unnamed(_) => quote! {(#construct_fields)}, + syn::Fields::Unit => quote! {}, + }; + + quote! { + #bound_fields + #compact #construct_fields + } +} +/// Construct original version of the struct from the compacted one +fn construct_decompact( + // The original structure to construct, i.e. struct name or enum variant path + original: &TokenStream2, + variant: &synstructure::VariantInfo<'_>, +) -> TokenStream2 { + let bound_fields: TokenStream2 = variant + .bindings() + .iter() + .map(|binding| { + let attrs = FieldAttrs::parse(&binding.ast().attrs); + let ty = &binding.ast().ty; + let cname = format_ident!("{}_c", binding.binding); + + match attrs { + FieldAttrs::Skip => quote! { + let #cname = Default::default(); + }, + FieldAttrs::Normal => { + quote! { + let #cname = <#ty as Compactable>::decompact(#binding, reg); + } + } + FieldAttrs::Registry(registry) => { + let reg_ident = format_ident!("{}", registry); + quote! { + let raw: ::Type = reg.read( + #binding + ); + let #cname = raw.into(); + } + } + } + }) + .collect(); + + let construct_fields: TokenStream2 = variant + .bindings() + .iter() + .map(|binding| { + let cname = format_ident!("{}_c", binding.binding); + if let Some(fname) = &binding.ast().ident { + quote! { #fname: #cname, } + } else { + quote! { #cname, } + } + }) + .collect(); + + let construct_fields = match variant.ast().fields { + syn::Fields::Named(_) => quote! {{ #construct_fields }}, + syn::Fields::Unnamed(_) => quote! {(#construct_fields)}, + syn::Fields::Unit => quote! {}, + }; + + quote! { + #bound_fields + #original #construct_fields + } +} + +// Sum of Compactable::count() of all fields. +fn sum_counts(variant: &synstructure::VariantInfo<'_>) -> TokenStream2 { + variant + .bindings() + .iter() + .map(|binding| { + let attrs = FieldAttrs::parse(&binding.ast().attrs); + let ty = &binding.ast().ty; + + match attrs { + FieldAttrs::Skip => quote! { CountPerTable::default() }, + FieldAttrs::Normal => { + quote! { <#ty as Compactable>::count(&#binding) } + } + FieldAttrs::Registry(registry) => { + let reg_ident = format_ident!("{}", registry); + quote! { + CountPerTable::#reg_ident(1) + } + } + } + }) + .fold( + quote! { CountPerTable::default() }, + |acc, x| quote! { #acc + #x }, + ) +} + +/// Generate a match arm for each variant of the compacted structure +/// using the given function to generate the pattern body. +fn each_variant_compact) -> TokenStream2>( + s: &synstructure::Structure, + compact_name: &TokenStream2, + mut f: F, +) -> TokenStream2 { + s.variants() + .iter() + .map(|variant| { + // Modify the binding pattern to match the compact variant + let mut v2 = variant.clone(); + v2.filter(|field| { + let attrs = FieldAttrs::parse(&field.ast().attrs); + !matches!(attrs, FieldAttrs::Skip) + }); + v2.bindings_mut().iter_mut().for_each(|binding| { + binding.style = synstructure::BindStyle::Move; + }); + let mut p = v2.pat().into_iter(); + let _ = p.next().expect("pattern always begins with an identifier"); + let p = quote! { #compact_name #(#p)* }; + + let decompacted = f(variant); + quote! { + #p => { #decompacted } + } + }) + .collect() +} + +/// Derives `Compact` trait for the given `struct` or `enum`. +pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { + s.add_bounds(synstructure::AddBounds::None) + .underscore_const(true); + + let name = &s.ast().ident; + let compact_name = format_ident!("Compact{}", name); + + let g = s.ast().generics.clone(); + let w = g.where_clause.clone(); + let def = match &s.ast().data { + syn::Data::Struct(v) => { + let variant: &synstructure::VariantInfo = &s.variants()[0]; + let defs = field_defs(&variant.ast().fields); + let semi = match v.fields { + syn::Fields::Named(_) => quote! {}, + syn::Fields::Unnamed(_) => quote! {;}, + syn::Fields::Unit => quote! {;}, + }; + quote! { + #[derive(Clone, serde::Serialize, serde::Deserialize)] + #[doc = concat!("Compacted version of `", stringify!(#name), "`.")] + pub struct #compact_name #g #w #defs #semi + } + } + syn::Data::Enum(_) => { + let variant_defs: TokenStream2 = s + .variants() + .iter() + .map(|variant| { + let vname = variant.ast().ident.clone(); + let defs = field_defs(&variant.ast().fields); + quote! { + #vname #defs, + } + }) + .collect(); + + quote! { + #[derive(Clone, serde::Serialize, serde::Deserialize)] + #[doc = concat!("Compacted version of `", stringify!(#name), "`.")] + pub enum #compact_name #g #w { #variant_defs } + } + } + syn::Data::Union(_) => panic!("unions are not supported"), + }; + + let count_per_variant = s.each_variant(|variant| sum_counts(variant)); + let construct_per_variant = s.each_variant(|variant| { + let vname = variant.ast().ident.clone(); + let construct = match &s.ast().data { + syn::Data::Struct(_) => quote! { #compact_name }, + syn::Data::Enum(_) => quote! {#compact_name :: #vname }, + syn::Data::Union(_) => unreachable!(), + }; + construct_compact(&construct, variant) + }); + + let decompact_per_variant = + each_variant_compact(&s, "e! {#compact_name}, |variant| { + let vname = variant.ast().ident.clone(); + let construct = match &s.ast().data { + syn::Data::Struct(_) => quote! { #name }, + syn::Data::Enum(_) => quote! {#name :: #vname }, + syn::Data::Union(_) => unreachable!(), + }; + construct_decompact(&construct, variant) + }); + + let impls = s.gen_impl(quote! { + use ::fuel_compression::{RegistryDb, tables, Table, Key, Compactable, CountPerTable, CompactionContext}; + + gen impl Compactable for @Self { + type Compact = #compact_name #g; + + fn count(&self) -> CountPerTable { + match self { #count_per_variant } + } + + fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact { + match self { #construct_per_variant } + } + + fn decompact(compact: Self::Compact, reg: &R) -> Self { + match compact { #decompact_per_variant } + } + } + }); + let rs = quote! { + #def + #impls + }; + + let _ = std::fs::write(format!("/tmp/derive/{}.rs", name), &rs.to_string()); + + rs +} diff --git a/fuel-derive/src/deserialize.rs b/fuel-derive/src/deserialize.rs index 0edbfb17e5..5c92a8e4c5 100644 --- a/fuel-derive/src/deserialize.rs +++ b/fuel-derive/src/deserialize.rs @@ -4,7 +4,7 @@ use quote::{ quote, }; -use crate::attribute::{ +use crate::canonical_attribute::{ should_skip_field, should_skip_field_binding, StructAttrs, diff --git a/fuel-derive/src/lib.rs b/fuel-derive/src/lib.rs index d130ba6b46..be1a0f856f 100644 --- a/fuel-derive/src/lib.rs +++ b/fuel-derive/src/lib.rs @@ -3,14 +3,17 @@ #![deny(unused_must_use, missing_docs)] extern crate proc_macro; -mod attribute; +mod canonical_attribute; +mod compact; mod deserialize; mod serialize; use self::{ + compact::compact_derive, deserialize::deserialize_derive, serialize::serialize_derive, }; + synstructure::decl_derive!( [Deserialize, attributes(canonical)] => /// Derives `Deserialize` trait for the given `struct` or `enum`. @@ -21,3 +24,8 @@ synstructure::decl_derive!( /// Derives `Serialize` trait for the given `struct` or `enum`. serialize_derive ); +synstructure::decl_derive!( + [Compact, attributes(da_compress)] => + /// Derives `Compact` trait for the given `struct` or `enum`. + compact_derive +); diff --git a/fuel-derive/src/serialize.rs b/fuel-derive/src/serialize.rs index 5a17dec106..00d5e1e689 100644 --- a/fuel-derive/src/serialize.rs +++ b/fuel-derive/src/serialize.rs @@ -1,7 +1,7 @@ use proc_macro2::TokenStream as TokenStream2; use quote::quote; -use crate::attribute::{ +use crate::canonical_attribute::{ should_skip_field_binding, StructAttrs, }; diff --git a/fuel-tx/Cargo.toml b/fuel-tx/Cargo.toml index 31112062f3..e79520db8f 100644 --- a/fuel-tx/Cargo.toml +++ b/fuel-tx/Cargo.toml @@ -15,6 +15,7 @@ bitflags = { workspace = true } derivative = { version = "2.2.0", default-features = false, features = ["use_core"], optional = true } derive_more = { version = "0.99", default-features = false, features = ["display"] } fuel-asm = { workspace = true, default-features = false } +fuel-compression = { workspace = true, optional = true } fuel-crypto = { workspace = true, default-features = false } fuel-merkle = { workspace = true, default-features = false, optional = true } fuel-types = { workspace = true, default-features = false } @@ -28,7 +29,6 @@ serde_json = { version = "1.0", default-features = false, features = ["alloc"], strum = { version = "0.24", default-features = false, optional = true } strum_macros = { version = "0.24", optional = true } wasm-bindgen = { version = "0.2.88", optional = true } -fuel-core-compression = { path = "../../fuel-core/crates/compression" } [dev-dependencies] bincode = { workspace = true } @@ -54,3 +54,4 @@ std = ["alloc", "fuel-asm/std", "fuel-crypto/std", "fuel-merkle/std", "fuel-type alloc = ["hashbrown", "fuel-types/alloc", "itertools/use_alloc", "derivative", "fuel-merkle", "strum", "strum_macros"] # serde is requiring alloc because its mandatory for serde_json. to avoid adding a new feature only for serde_json, we just require `alloc` here since as of the moment we don't have a use case of serde without alloc. serde = ["alloc", "dep:serde", "fuel-asm/serde", "fuel-crypto/serde", "fuel-types/serde", "fuel-merkle/serde", "serde_json", "hashbrown/serde", "bitflags/serde"] +da-compression = ["serde", "fuel-compression", "fuel-types/da-compression"] diff --git a/fuel-tx/src/lib.rs b/fuel-tx/src/lib.rs index 341e0dc1bb..b7a843a641 100644 --- a/fuel-tx/src/lib.rs +++ b/fuel-tx/src/lib.rs @@ -101,6 +101,9 @@ pub use transaction::{ Witness, }; +#[cfg(feature = "da-compression")] +pub use transaction::CompactTransaction; + pub use transaction::{ Signable, UniqueIdentifier, diff --git a/fuel-tx/src/transaction.rs b/fuel-tx/src/transaction.rs index 7b0d8d2013..ab0699ced3 100644 --- a/fuel-tx/src/transaction.rs +++ b/fuel-tx/src/transaction.rs @@ -90,6 +90,7 @@ pub type TxId = Bytes32; /// The fuel transaction entity . #[derive(Debug, Clone, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[allow(clippy::large_enum_variant)] #[non_exhaustive] pub enum Transaction { diff --git a/fuel-tx/src/transaction/policies.rs b/fuel-tx/src/transaction/policies.rs index 0e6f59723e..3750dc3d8a 100644 --- a/fuel-tx/src/transaction/policies.rs +++ b/fuel-tx/src/transaction/policies.rs @@ -20,11 +20,14 @@ use rand::{ Rng, }; +/// See https://github.com/FuelLabs/fuel-specs/blob/master/src/tx-format/policy.md#policy +#[derive(Clone, Copy, Default, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +pub struct PoliciesBits(u32); + bitflags::bitflags! { - /// See https://github.com/FuelLabs/fuel-specs/blob/master/src/tx-format/policy.md#policy - #[derive(Clone, Copy, Default, Debug, PartialEq, Eq, Hash)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - pub struct PoliciesBits: u32 { + impl PoliciesBits: u32 { /// If set, the gas price is present in the policies. const GasPrice = 1 << 0; /// If set, the witness limit is present in the policies. @@ -82,8 +85,8 @@ pub const POLICIES_NUMBER: usize = PoliciesBits::all().bits().count_ones() as us /// Container for managing policies. #[derive(Clone, Copy, Default, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct Policies { /// A bitmask that indicates what policies are set. bits: PoliciesBits, diff --git a/fuel-tx/src/transaction/types/create.rs b/fuel-tx/src/transaction/types/create.rs index e5155ae055..fb3aaca94b 100644 --- a/fuel-tx/src/transaction/types/create.rs +++ b/fuel-tx/src/transaction/types/create.rs @@ -106,6 +106,7 @@ impl CreateMetadata { #[derive(Default, Debug, Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Create)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] @@ -120,6 +121,7 @@ pub struct Create { pub(crate) witnesses: Vec, pub(crate) salt: Salt, #[cfg_attr(feature = "serde", serde(skip))] + #[cfg_attr(feature = "da-compression", da_compress(skip))] #[derivative(PartialEq = "ignore", Hash = "ignore")] #[canonical(skip)] pub(crate) metadata: Option, diff --git a/fuel-tx/src/transaction/types/input.rs b/fuel-tx/src/transaction/types/input.rs index 5402d3d5b4..96419cd2bd 100644 --- a/fuel-tx/src/transaction/types/input.rs +++ b/fuel-tx/src/transaction/types/input.rs @@ -13,6 +13,8 @@ use core::{ fmt, fmt::Formatter, }; +#[cfg(feature = "da-compression")] +use fuel_compression::Compactable; use fuel_crypto::{ Hasher, PublicKey, @@ -100,6 +102,29 @@ impl Deserialize for Empty { } } +#[cfg(feature = "da-compression")] +impl Compactable for Empty +where + T: Compactable, +{ + type Compact = (); + + fn count(&self) -> fuel_compression::CountPerTable { + Default::default() + } + + fn compact( + &self, + _: &mut fuel_compression::CompactionContext, + ) -> Self::Compact { + () + } + + fn decompact(_: Self::Compact, _: &R) -> Self { + Self(Default::default()) + } +} + impl AsFieldFmt for Empty { fn fmt_as_field(&self, f: &mut Formatter) -> fmt::Result { f.write_str("Empty") @@ -181,7 +206,7 @@ where #[derive(Debug, Clone, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[non_exhaustive] pub enum Input { CoinSigned(CoinSigned), diff --git a/fuel-tx/src/transaction/types/input/coin.rs b/fuel-tx/src/transaction/types/input/coin.rs index 8a32c592c4..82bb13e6fc 100644 --- a/fuel-tx/src/transaction/types/input/coin.rs +++ b/fuel-tx/src/transaction/types/input/coin.rs @@ -11,6 +11,8 @@ use crate::{ }; use alloc::vec::Vec; use derivative::Derivative; +#[cfg(feature = "da-compression")] +use fuel_compression::Compactable; use fuel_types::{ Address, AssetId, @@ -31,6 +33,14 @@ mod private { } /// Specifies the coin based on the usage context. See [`Coin`]. +#[cfg(feature = "da-compression")] +pub trait CoinSpecification: private::Seal { + type Witness: AsField + Compactable + Clone; + type Predicate: AsField> + Compactable + Clone; + type PredicateData: AsField> + Compactable + Clone; + type PredicateGasUsed: AsField + Compactable + Clone; +} +#[cfg(not(feature = "da-compression"))] pub trait CoinSpecification: private::Seal { type Witness: AsField; type Predicate: AsField>; @@ -40,6 +50,7 @@ pub trait CoinSpecification: private::Seal { #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] pub struct Signed; impl CoinSpecification for Signed { @@ -51,6 +62,7 @@ impl CoinSpecification for Signed { #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] pub struct Predicate; impl CoinSpecification for Predicate { @@ -96,17 +108,17 @@ impl CoinSpecification for Full { #[derive(Default, Derivative, Clone, PartialEq, Eq, Hash)] #[derivative(Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct Coin where - Specification: CoinSpecification, + Specification: CoinSpecification + Clone, { pub utxo_id: UtxoId, - #[da_compress(registry = "Address")] + #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] pub owner: Address, pub amount: Word, - #[da_compress(registry = "AssetId")] + #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] pub asset_id: AssetId, pub tx_pointer: TxPointer, #[derivative(Debug(format_with = "fmt_as_field"))] @@ -122,7 +134,7 @@ where impl Coin where - Specification: CoinSpecification, + Specification: CoinSpecification + Clone, { /// The "Note" section from the specification: /// . diff --git a/fuel-tx/src/transaction/types/input/contract.rs b/fuel-tx/src/transaction/types/input/contract.rs index 178c07a702..56f70826ab 100644 --- a/fuel-tx/src/transaction/types/input/contract.rs +++ b/fuel-tx/src/transaction/types/input/contract.rs @@ -14,18 +14,18 @@ use fuel_types::{ /// the `fuel-vm`. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen(js_name = InputContract))] pub struct Contract { pub utxo_id: UtxoId, - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub balance_root: Bytes32, - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub state_root: Bytes32, - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub tx_pointer: TxPointer, - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub contract_id: ContractId, } diff --git a/fuel-tx/src/transaction/types/input/message.rs b/fuel-tx/src/transaction/types/input/message.rs index 2906c63422..635115a94f 100644 --- a/fuel-tx/src/transaction/types/input/message.rs +++ b/fuel-tx/src/transaction/types/input/message.rs @@ -4,6 +4,8 @@ use crate::{ }; use alloc::vec::Vec; use derivative::Derivative; +#[cfg(feature = "da-compression")] +use fuel_compression::Compactable; use fuel_types::{ Address, MessageId, @@ -30,6 +32,36 @@ mod private { } /// Specifies the message based on the usage context. See [`Message`]. +#[cfg(feature = "da-compression")] +pub trait MessageSpecification: private::Seal { + type Data: AsField> + + Compactable + + Clone + + serde::Serialize + + for<'a> serde::Deserialize<'a>; + type Predicate: AsField> + + Compactable + + Clone + + serde::Serialize + + for<'a> serde::Deserialize<'a>; + type PredicateData: AsField> + + Compactable + + Clone + + serde::Serialize + + for<'a> serde::Deserialize<'a>; + type PredicateGasUsed: AsField + + Compactable + + Clone + + serde::Serialize + + for<'a> serde::Deserialize<'a> + + Default; + type Witness: AsField + + Compactable + + Clone + + serde::Serialize + + for<'a> serde::Deserialize<'a>; +} +#[cfg(not(feature = "da-compression"))] pub trait MessageSpecification: private::Seal { type Data: AsField>; type Predicate: AsField>; @@ -50,12 +82,14 @@ pub mod specifications { /// `witnesses` vector of the [`crate::Transaction`]. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] pub struct Signed; /// The type means that the message is not signed, and the `owner` is a `predicate` /// bytecode. The merkle root from the `predicate` should be equal to the `owner`. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] pub struct Predicate; /// The retrayable message metadata. It is a message that can't be used as a coin to @@ -65,7 +99,6 @@ pub mod specifications { /// is not consumed and can be used later until successful execution. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct MessageData(core::marker::PhantomData); impl MessageSpecification for MessageData { @@ -144,24 +177,24 @@ pub mod specifications { #[derive(Default, Derivative, Clone, PartialEq, Eq, Hash)] #[derivative(Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct Message where - Specification: MessageSpecification, + Specification: MessageSpecification + Clone, { /// The sender from the L1 chain. - #[da_compress(registry = "Address")] + #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] pub sender: Address, /// The receiver on the `Fuel` chain. - #[da_compress(registry = "Address")] + #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] pub recipient: Address, pub amount: Word, pub nonce: Nonce, #[derivative(Debug(format_with = "fmt_as_field"))] pub witness_index: Specification::Witness, #[derivative(Debug(format_with = "fmt_as_field"))] - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub predicate_gas_used: Specification::PredicateGasUsed, #[derivative(Debug(format_with = "fmt_as_field"))] pub data: Specification::Data, @@ -173,7 +206,7 @@ where impl Message where - Specification: MessageSpecification, + Specification: MessageSpecification + Clone, { pub fn prepare_sign(&mut self) { if let Some(predicate_gas_used_field) = self.predicate_gas_used.as_mut_field() { diff --git a/fuel-tx/src/transaction/types/mint.rs b/fuel-tx/src/transaction/types/mint.rs index f0c30aade8..e20f46d28f 100644 --- a/fuel-tx/src/transaction/types/mint.rs +++ b/fuel-tx/src/transaction/types/mint.rs @@ -49,14 +49,14 @@ impl MintMetadata { /// by it. #[derive(Default, Debug, Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[canonical(prefix = TransactionRepr::Mint)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derivative(Eq, PartialEq, Hash)] pub struct Mint { /// The location of the transaction in the block. - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub(crate) tx_pointer: TxPointer, /// The `Input::Contract` that assets are minted to. pub(crate) input_contract: input::contract::Contract, @@ -65,12 +65,12 @@ pub struct Mint { /// The amount of funds minted. pub(crate) mint_amount: Word, /// The asset IDs corresponding to the minted amount. - #[da_compress(registry = "AssetId")] + #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] pub(crate) mint_asset_id: AssetId, #[cfg_attr(feature = "serde", serde(skip))] #[derivative(PartialEq = "ignore", Hash = "ignore")] #[canonical(skip)] - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub(crate) metadata: Option, } diff --git a/fuel-tx/src/transaction/types/output.rs b/fuel-tx/src/transaction/types/output.rs index 8e1bac754a..330b605fbd 100644 --- a/fuel-tx/src/transaction/types/output.rs +++ b/fuel-tx/src/transaction/types/output.rs @@ -1,6 +1,9 @@ use fuel_crypto::Hasher; use fuel_types::{ - canonical, + canonical::{ + self, + Serialize as _, + }, Address, AssetId, Bytes32, @@ -9,8 +12,6 @@ use fuel_types::{ Word, }; -use core::mem; - mod consts; pub mod contract; mod repr; @@ -20,31 +21,35 @@ pub use repr::OutputRepr; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(canonical::Deserialize, canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[non_exhaustive] pub enum Output { Coin { + #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] to: Address, amount: Word, + #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] asset_id: AssetId, }, Contract(Contract), Change { + #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] to: Address, - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] amount: Word, + #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] asset_id: AssetId, }, Variable { - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] to: Address, - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] amount: Word, - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] asset_id: AssetId, }, diff --git a/fuel-tx/src/transaction/types/output/contract.rs b/fuel-tx/src/transaction/types/output/contract.rs index d1c6ad141d..80fad07cc4 100644 --- a/fuel-tx/src/transaction/types/output/contract.rs +++ b/fuel-tx/src/transaction/types/output/contract.rs @@ -7,17 +7,17 @@ use fuel_types::Bytes32; /// the `fuel-vm`. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen(js_name = OutputContract))] pub struct Contract { /// Index of input contract. pub input_index: u8, /// Root of amount of coins owned by contract after transaction execution. - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub balance_root: Bytes32, /// State root of contract after transaction execution. - #[da_compress(skip)] + #[cfg_attr(feature = "da-compression", da_compress(skip))] pub state_root: Bytes32, } diff --git a/fuel-tx/src/transaction/types/script.rs b/fuel-tx/src/transaction/types/script.rs index e66d8ea1f7..7cdfbca8bf 100644 --- a/fuel-tx/src/transaction/types/script.rs +++ b/fuel-tx/src/transaction/types/script.rs @@ -55,13 +55,13 @@ pub(crate) struct ScriptMetadata { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[canonical(prefix = TransactionRepr::Script)] #[derivative(Eq, PartialEq, Hash, Debug)] pub struct Script { pub(crate) script_gas_limit: Word, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] - #[da_compress(registry = "ScriptCode")] + #[cfg_attr(feature = "da-compression", da_compress(registry = "ScriptCode"))] pub(crate) script: Vec, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] pub(crate) script_data: Vec, @@ -71,9 +71,9 @@ pub struct Script { pub(crate) witnesses: Vec, pub(crate) receipts_root: Bytes32, #[cfg_attr(feature = "serde", serde(skip))] + #[cfg_attr(feature = "da-compression", da_compress(skip))] #[derivative(PartialEq = "ignore", Hash = "ignore")] #[canonical(skip)] - #[da_compress(skip)] pub(crate) metadata: Option, } diff --git a/fuel-tx/src/transaction/types/storage.rs b/fuel-tx/src/transaction/types/storage.rs index df50ae8e39..9ad9244b22 100644 --- a/fuel-tx/src/transaction/types/storage.rs +++ b/fuel-tx/src/transaction/types/storage.rs @@ -20,6 +20,7 @@ use core::cmp::Ordering; #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derive(Deserialize, Serialize)] pub struct StorageSlot { diff --git a/fuel-tx/src/transaction/types/utxo_id.rs b/fuel-tx/src/transaction/types/utxo_id.rs index e089e4e104..8031173912 100644 --- a/fuel-tx/src/transaction/types/utxo_id.rs +++ b/fuel-tx/src/transaction/types/utxo_id.rs @@ -20,8 +20,8 @@ use rand::{ #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct UtxoId { /// transaction id tx_id: TxId, @@ -29,6 +29,29 @@ pub struct UtxoId { output_index: u8, } +// const _: () = { +// impl fuel_compression::Compactable for UtxoId { +// type Compact = (); + +// fn count(&self) -> fuel_compression::CountPerTable { +// todo!() +// } + +// fn compact(&self, ctx: &mut fuel_compression::CompactionContext) -> +// Self::Compact where +// R: fuel_compression::db::RegistryRead + +// fuel_compression::db::RegistryWrite + fuel_compression::db::RegistryIndex { +// todo!() +// } + +// fn decompact(compact: Self::Compact, reg: &R) -> Self +// where +// R: fuel_compression::db::RegistryRead { +// todo!() +// } +// } +// }; + impl UtxoId { pub const LEN: usize = TxId::LEN + 8; diff --git a/fuel-tx/src/transaction/types/witness.rs b/fuel-tx/src/transaction/types/witness.rs index 611f7aa697..cde5bbb25a 100644 --- a/fuel-tx/src/transaction/types/witness.rs +++ b/fuel-tx/src/transaction/types/witness.rs @@ -26,9 +26,11 @@ use rand::{ #[derivative(Debug)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] pub struct Witness { #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "Witness"))] data: Vec, } diff --git a/fuel-tx/src/tx_pointer.rs b/fuel-tx/src/tx_pointer.rs index 95872e5666..0153b2111e 100644 --- a/fuel-tx/src/tx_pointer.rs +++ b/fuel-tx/src/tx_pointer.rs @@ -26,6 +26,7 @@ use rand::{ #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(Deserialize, Serialize)] pub struct TxPointer { /// Block height diff --git a/fuel-types/Cargo.toml b/fuel-types/Cargo.toml index 7695a0c818..5de786139c 100644 --- a/fuel-types/Cargo.toml +++ b/fuel-types/Cargo.toml @@ -11,7 +11,7 @@ repository = { workspace = true } description = "Atomic types of the FuelVM." [dependencies] -fuel-core-compression = { path = "../../fuel-core/crates/compression" } +fuel-compression = { workspace = true } fuel-derive = { workspace = true } hex = { version = "0.4", default-features = false, optional = true } rand = { version = "0.8", default-features = false, optional = true } @@ -32,6 +32,7 @@ typescript = ["wasm-bindgen"] alloc = ["hex/alloc"] random = ["rand"] serde = ["dep:serde", "alloc"] +da-compression = ["serde"] std = ["alloc", "serde?/std", "hex?/std"] unsafe = [] diff --git a/fuel-types/src/array_types.rs b/fuel-types/src/array_types.rs index d6edf80685..d137bebe8b 100644 --- a/fuel-types/src/array_types.rs +++ b/fuel-types/src/array_types.rs @@ -35,10 +35,10 @@ macro_rules! key { /// FuelVM atomic array type. #[repr(transparent)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, )] - #[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct $i([u8; $s]); key_methods!($i, $s); @@ -58,10 +58,10 @@ macro_rules! key_with_big_array { #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, )] - #[derive(fuel_core_compression::Serialize, fuel_core_compression::Deserialize)] pub struct $i([u8; $s]); key_methods!($i, $s); diff --git a/fuel-types/src/numeric_types.rs b/fuel-types/src/numeric_types.rs index 50c25fb36b..838475e1f2 100644 --- a/fuel-types/src/numeric_types.rs +++ b/fuel-types/src/numeric_types.rs @@ -34,6 +34,7 @@ macro_rules! key { #[repr(transparent)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(transparent))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index e271276ae0..546f271823 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -20,6 +20,7 @@ derive_more = { version = "0.99", default-features = false, features = ["display dyn-clone = { version = "1.0", optional = true } ethnum = "1.3" fuel-asm = { workspace = true, default-features = false } +fuel-compression = { workspace = true, default-features = false, optional = true} fuel-crypto = { workspace = true, default-features = false } fuel-merkle = { workspace = true, default-features = false } fuel-storage = { workspace = true } @@ -64,5 +65,6 @@ profile-gas = ["profile-any"] profile-coverage = ["profile-any"] profile-any = ["dyn-clone"] # All profiling features should depend on this random = ["fuel-crypto/random", "fuel-types/random", "fuel-tx/random", "rand"] +da-compression = ["fuel-compression", "fuel-tx/da-compression"] serde = ["dep:serde", "hashbrown/serde", "fuel-asm/serde", "fuel-types/serde", "fuel-tx/serde", "fuel-merkle/serde", "backtrace?/serde"] -test-helpers = ["fuel-tx/builder", "alloc", "random", "dep:anyhow", "fuel-crypto/test-helpers"] +test-helpers = ["fuel-tx/builder", "alloc", "random", "dep:anyhow", "fuel-crypto/test-helpers", "fuel-compression/test-helpers"] diff --git a/fuel-vm/src/lib.rs b/fuel-vm/src/lib.rs index abac9c0152..1463745862 100644 --- a/fuel-vm/src/lib.rs +++ b/fuel-vm/src/lib.rs @@ -59,6 +59,9 @@ pub mod profiler { #[doc(no_inline)] pub use fuel_asm; #[doc(no_inline)] +#[cfg(feature = "da-compression")] +pub use fuel_compression; +#[doc(no_inline)] pub use fuel_crypto; #[doc(no_inline)] pub use fuel_merkle; From 0c7f3dd2dc77b3b9220a3dc449e5e50bdaae923c Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 5 Feb 2024 18:27:29 +0200 Subject: [PATCH 03/77] Add doc comments and cleanup --- fuel-compression/Cargo.toml | 10 +++------- fuel-compression/src/compaction.rs | 4 ++++ fuel-compression/src/lib.rs | 8 ++++++++ fuel-compression/src/registry/db.rs | 1 + fuel-compression/src/registry/key.rs | 2 ++ fuel-compression/src/registry/mod.rs | 10 ++++++++++ 6 files changed, 28 insertions(+), 7 deletions(-) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index 4bab089d9d..acdce1c57e 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -11,17 +11,13 @@ repository = { workspace = true } description = "Compression and decompression of Fuel blocks for DA storage." [dependencies] +fuel-derive = { workspace = true } +postcard = { version = "1.0", features = ["use-std"] } serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" -postcard = { version = "1.0", features = ["use-std"] } -bincode = "1.3" - -paste = "1.0" - -fuel-derive = { workspace = true } - [dev-dependencies] +bincode = "1.3" fuel-compression = { path = "." } # Self-dependency needed by test for macros fuel-asm = { workspace = true } fuel-types = { workspace = true, features = ["serde"] } diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 29433f303e..d819017c8c 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -23,6 +23,7 @@ use crate::{ Key, }; +/// Context for compaction, i.e. converting data to reference-based format #[must_use] pub struct CompactionContext<'a, R> { /// The registry @@ -102,13 +103,16 @@ impl<'a, R: RegistryDb> CompactionContext<'a, R> { /// Convert data to reference-based format pub trait Compactable { + /// The compacted type with references type Compact: Clone + Serialize + for<'a> Deserialize<'a>; /// Count max number of each key type, for upper limit of overwritten keys fn count(&self) -> CountPerTable; + /// Convert to compacted format fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact; + /// Convert from compacted format fn decompact(compact: Self::Compact, reg: &R) -> Self; } diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index 25967d6295..c2cc9d00d3 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -1,3 +1,11 @@ +//! Compression and decompression of fuel-types for the DA layer + +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![warn(missing_docs)] +#![deny(unsafe_code)] +#![deny(unused_crate_dependencies)] +#![deny(clippy::cast_possible_truncation)] + mod compaction; mod registry; diff --git a/fuel-compression/src/registry/db.rs b/fuel-compression/src/registry/db.rs index 40c729c31e..9180ad172c 100644 --- a/fuel-compression/src/registry/db.rs +++ b/fuel-compression/src/registry/db.rs @@ -3,6 +3,7 @@ use super::{ Table, }; +/// Registry database needs these behaviors pub trait RegistryDb { /// Get next key for the given table. This is where the next write should start at. /// The result of this function is just a suggestion, and the caller may choose to diff --git a/fuel-compression/src/registry/key.rs b/fuel-compression/src/registry/key.rs index 0b4379ab69..9da4df9528 100644 --- a/fuel-compression/src/registry/key.rs +++ b/fuel-compression/src/registry/key.rs @@ -109,10 +109,12 @@ impl Key { /// This is the first writable key. pub const ZERO: Self = Self(RawKey::ZERO, PhantomData); + /// Obtain untyped key. pub fn raw(&self) -> RawKey { self.0 } + /// Construct from untyped key. pub fn from_raw(raw: RawKey) -> Self { Self(raw, PhantomData) } diff --git a/fuel-compression/src/registry/mod.rs b/fuel-compression/src/registry/mod.rs index 7b9486dd40..b5a1a5fb3d 100644 --- a/fuel-compression/src/registry/mod.rs +++ b/fuel-compression/src/registry/mod.rs @@ -18,8 +18,12 @@ mod _private { pub trait Seal {} } +/// Table in the registry pub trait Table: _private::Seal { + /// Unique name of the table const NAME: &'static str; + + /// The type stored in the table type Type: PartialEq + Default + Serialize + for<'de> Deserialize<'de>; } @@ -40,6 +44,7 @@ pub mod access { macro_rules! tables { // $index muse use increasing numbers starting from zero ($($name:ident: $ty:ty),*$(,)?) => { + /// Marker struct for each table type pub mod tables { $( /// Specifies the table to use for a given key. @@ -58,6 +63,7 @@ macro_rules! tables { /// One counter per table #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] #[allow(non_snake_case)] // The field names match table type names eactly + #[allow(missing_docs)] // Makes no sense to document the fields #[non_exhaustive] pub struct CountPerTable { $(pub $name: usize),* @@ -101,6 +107,7 @@ macro_rules! tables { /// One key value per table #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[allow(non_snake_case)] // The field names match table type names eactly + #[allow(missing_docs)] // Makes no sense to document the fields #[non_exhaustive] pub struct KeyPerTable { $(pub $name: Key),* @@ -152,16 +159,19 @@ macro_rules! tables { /// Registeration changes per table #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[allow(non_snake_case)] // The field names match table type names eactly + #[allow(missing_docs)] // Makes no sense to document the fields #[non_exhaustive] pub struct ChangesPerTable { $(pub $name: WriteTo),* } impl ChangesPerTable { + /// Is the changeset empty for all tables? pub fn is_empty(&self) -> bool { true $(&& self.$name.values.is_empty())* } + /// Create a new changeset with the given start keys pub fn from_start_keys(start_keys: KeyPerTable) -> Self { Self { $($name: WriteTo { From cbc21735690f36200fdc760cec4e47bbe8e91e7f Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 5 Feb 2024 18:51:30 +0200 Subject: [PATCH 04/77] Add changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5403b56ede..28e7a599b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added + +- [#670](https://github.com/FuelLabs/fuel-vm/pull/670): Add DA compression functionality to `Transaction` and any types within + ## [Version 0.45.0] ### Changed From 82b2efa0b0cb554ef01538ae856ac928a9d0f23d Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 5 Feb 2024 18:52:27 +0200 Subject: [PATCH 05/77] Cargo.toml fmt --- fuel-compression/Cargo.toml | 4 ++-- fuel-vm/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index acdce1c57e..309c4afafd 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -18,10 +18,10 @@ serde-big-array = "0.5" [dev-dependencies] bincode = "1.3" -fuel-compression = { path = "." } # Self-dependency needed by test for macros fuel-asm = { workspace = true } -fuel-types = { workspace = true, features = ["serde"] } +fuel-compression = { path = "." } # Self-dependency needed by test for macros fuel-tx = { workspace = true } +fuel-types = { workspace = true, features = ["serde"] } [features] test-helpers = [] diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index 546f271823..8448b09cf3 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -20,7 +20,7 @@ derive_more = { version = "0.99", default-features = false, features = ["display dyn-clone = { version = "1.0", optional = true } ethnum = "1.3" fuel-asm = { workspace = true, default-features = false } -fuel-compression = { workspace = true, default-features = false, optional = true} +fuel-compression = { workspace = true, default-features = false, optional = true } fuel-crypto = { workspace = true, default-features = false } fuel-merkle = { workspace = true, default-features = false } fuel-storage = { workspace = true } From 1e792e9014a3986d5c4e9c027b3d52c717604e96 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 12 Feb 2024 18:47:42 +0200 Subject: [PATCH 06/77] Wrap Compaction return types to anyhow::Result --- Cargo.toml | 1 + fuel-compression/Cargo.toml | 1 + fuel-compression/src/compaction.rs | 100 +++++++++++++++------ fuel-compression/src/registry/db.rs | 12 ++- fuel-compression/src/registry/in_memory.rs | 43 ++++++--- fuel-compression/src/registry/mod.rs | 13 +-- fuel-derive/src/compact.rs | 16 ++-- fuel-tx/Cargo.toml | 3 +- fuel-tx/src/transaction/types/input.rs | 11 ++- fuel-types/Cargo.toml | 3 +- fuel-vm/Cargo.toml | 4 +- 11 files changed, 140 insertions(+), 67 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index db3ce65d49..87103d5301 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ repository = "https://github.com/FuelLabs/fuel-vm" version = "0.45.0" [workspace.dependencies] +anyhow = "1.0" fuel-asm = { version = "0.45.0", path = "fuel-asm", default-features = false } fuel-compression = { version = "0.45.0", path = "fuel-compression", default-features = false } fuel-crypto = { version = "0.45.0", path = "fuel-crypto", default-features = false } diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index 309c4afafd..8c1bd8791c 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } description = "Compression and decompression of Fuel blocks for DA storage." [dependencies] +anyhow = { workspace = true } fuel-derive = { workspace = true } postcard = { version = "1.0", features = ["use-std"] } serde = { version = "1.0", features = ["derive"] } diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index d819017c8c..d2d64e54a0 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -1,4 +1,7 @@ -use std::marker::PhantomData; +use std::{ + marker::PhantomData, + mem::MaybeUninit, +}; use serde::{ Deserialize, @@ -44,8 +47,8 @@ impl<'a, R: RegistryDb> CompactionContext<'a, R> { pub fn run( reg: &'a mut R, target: C, - ) -> (C::Compact, ChangesPerTable) { - let start_keys = next_keys(reg); + ) -> anyhow::Result<(C::Compact, ChangesPerTable)> { + let start_keys = next_keys(reg)?; let next_keys = start_keys; let key_limits = target.count(); let safe_keys_start = add_keys(next_keys, key_limits); @@ -58,16 +61,16 @@ impl<'a, R: RegistryDb> CompactionContext<'a, R> { changes: ChangesPerTable::from_start_keys(start_keys), }; - let compacted = target.compact(&mut ctx); - ctx.changes.apply_to_registry(ctx.reg); - (compacted, ctx.changes) + let compacted = target.compact(&mut ctx)?; + ctx.changes.apply_to_registry(ctx.reg)?; + Ok((compacted, ctx.changes)) } } impl<'a, R: RegistryDb> CompactionContext<'a, R> { /// Convert a value to a key /// If necessary, store the value in the changeset and allocate a new key. - pub fn to_key(&mut self, value: T::Type) -> Key + pub fn to_key(&mut self, value: T::Type) -> anyhow::Result> where KeyPerTable: access::AccessCopy>, KeyPerTable: access::AccessMut>, @@ -79,16 +82,16 @@ impl<'a, R: RegistryDb> CompactionContext<'a, R> { >>::get(&self.changes) .lookup_value(&value) { - return key; + return Ok(key); } // Check if the registry contains this value already - if let Some(key) = self.reg.index_lookup::(&value) { + if let Some(key) = self.reg.index_lookup::(&value)? { let start: Key = self.start_keys.value(); let end: Key = self.safe_keys_start.value(); // Check if the value is in the possibly-overwritable range if !key.is_between(start, end) { - return key; + return Ok(key); } } // Allocate a new key for this @@ -97,7 +100,7 @@ impl<'a, R: RegistryDb> CompactionContext<'a, R> { >>::get_mut(&mut self.changes) .values .push(value); - key + Ok(key) } } @@ -110,10 +113,15 @@ pub trait Compactable { fn count(&self) -> CountPerTable; /// Convert to compacted format - fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact; + fn compact( + &self, + ctx: &mut CompactionContext, + ) -> anyhow::Result; /// Convert from compacted format - fn decompact(compact: Self::Compact, reg: &R) -> Self; + fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result + where + Self: Sized; } macro_rules! identity_compaction { @@ -128,12 +136,15 @@ macro_rules! identity_compaction { fn compact( &self, _ctx: &mut CompactionContext, - ) -> Self::Compact { - *self + ) -> anyhow::Result { + Ok(*self) } - fn decompact(compact: Self::Compact, _reg: &R) -> Self { - compact + fn decompact( + compact: Self::Compact, + _reg: &R, + ) -> anyhow::Result { + Ok(compact) } } }; @@ -150,6 +161,25 @@ pub struct ArrayWrapper Deserialize<'a>>( #[serde(with = "serde_big_array::BigArray")] pub [T; S], ); +// TODO: use try_map when stabilized: https://github.com/rust-lang/rust/issues/79711 +#[allow(unsafe_code)] +fn try_map_array Result>( + arr: [T; S], + mut f: F, +) -> Result<[R; S], E> { + // SAFETY: we are claiming to have initialized an array of `MaybeUninit`s, + // which do not require initialization. + let mut tmp: [MaybeUninit; S] = unsafe { MaybeUninit::uninit().assume_init() }; + + // Dropping a `MaybeUninit` does nothing, so we can just overwrite the array. + for (i, v) in arr.into_iter().enumerate() { + tmp[i] = MaybeUninit::new(f(v)?); + } + + // SAFETY: Everything element is initialized. + Ok(tmp.map(|v| unsafe { v.assume_init() })) +} + impl Compactable for [T; S] where T: Compactable + Clone + Serialize + for<'a> Deserialize<'a>, @@ -164,12 +194,19 @@ where count } - fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact { - ArrayWrapper(self.clone().map(|item| item.compact(ctx))) + fn compact( + &self, + ctx: &mut CompactionContext, + ) -> anyhow::Result { + Ok(ArrayWrapper(try_map_array(self.clone(), |v: T| { + v.compact(ctx) + })?)) } - fn decompact(compact: Self::Compact, reg: &R) -> Self { - compact.0.map(|item| T::decompact(item, reg)) + fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result { + Ok(try_map_array(compact.0, |v: T::Compact| { + T::decompact(v, reg) + })?) } } @@ -187,11 +224,14 @@ where count } - fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact { + fn compact( + &self, + ctx: &mut CompactionContext, + ) -> anyhow::Result { self.iter().map(|item| item.compact(ctx)).collect() } - fn decompact(compact: Self::Compact, reg: &R) -> Self { + fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result { compact .into_iter() .map(|item| T::decompact(item, reg)) @@ -206,12 +246,18 @@ impl Compactable for PhantomData { CountPerTable::default() } - fn compact(&self, _ctx: &mut CompactionContext) -> Self::Compact { - () + fn compact( + &self, + _ctx: &mut CompactionContext, + ) -> anyhow::Result { + Ok(()) } - fn decompact(_compact: Self::Compact, _reg: &R) -> Self { - Self + fn decompact( + _compact: Self::Compact, + _reg: &R, + ) -> anyhow::Result { + Ok(Self) } } diff --git a/fuel-compression/src/registry/db.rs b/fuel-compression/src/registry/db.rs index 9180ad172c..66c694da7e 100644 --- a/fuel-compression/src/registry/db.rs +++ b/fuel-compression/src/registry/db.rs @@ -8,14 +8,18 @@ pub trait RegistryDb { /// Get next key for the given table. This is where the next write should start at. /// The result of this function is just a suggestion, and the caller may choose to /// ignore it, although it's rare that they would know better. - fn next_key(&self) -> Key; + fn next_key(&self) -> anyhow::Result>; /// Read a value from the registry by key - fn read(&self, key: Key) -> T::Type; + fn read(&self, key: Key) -> anyhow::Result; /// Write a continuous sequence of values to the registry - fn batch_write(&mut self, start_key: Key, values: Vec); + fn batch_write( + &mut self, + start_key: Key, + values: Vec, + ) -> anyhow::Result<()>; /// Lookup a key by value - fn index_lookup(&self, value: &T::Type) -> Option>; + fn index_lookup(&self, value: &T::Type) -> anyhow::Result>>; } diff --git a/fuel-compression/src/registry/in_memory.rs b/fuel-compression/src/registry/in_memory.rs index f55b001597..344b573af7 100644 --- a/fuel-compression/src/registry/in_memory.rs +++ b/fuel-compression/src/registry/in_memory.rs @@ -16,23 +16,30 @@ pub struct InMemoryRegistry { } impl RegistryDb for InMemoryRegistry { - fn next_key(&self) -> Key { - Key::from_raw(self.next_keys.get(T::NAME).copied().unwrap_or(RawKey::ZERO)) + fn next_key(&self) -> anyhow::Result> { + Ok(Key::from_raw( + self.next_keys.get(T::NAME).copied().unwrap_or(RawKey::ZERO), + )) } - fn read(&self, key: Key) -> T::Type { + fn read(&self, key: Key) -> anyhow::Result { if key == Key::DEFAULT_VALUE { - return T::Type::default(); + return Ok(T::Type::default()); } - self.storage + Ok(self + .storage .get(T::NAME) .and_then(|table| table.get(&key.raw())) .map(|bytes| postcard::from_bytes(bytes).expect("Invalid value in registry")) - .unwrap_or_default() + .unwrap_or_default()) } - fn batch_write(&mut self, start_key: Key, values: Vec) { + fn batch_write( + &mut self, + start_key: Key, + values: Vec, + ) -> anyhow::Result<()> { let empty = values.is_empty(); if !empty && start_key == Key::DEFAULT_VALUE { panic!("Cannot write to the default value key"); @@ -50,24 +57,32 @@ impl RegistryDb for InMemoryRegistry { if !empty { self.next_keys.insert(T::NAME, key); } + Ok(()) } - fn index_lookup(&self, value: &T::Type) -> Option> { + fn index_lookup(&self, value: &T::Type) -> anyhow::Result>> { if *value == T::Type::default() { - return Some(Key::DEFAULT_VALUE); + return Ok(Some(Key::DEFAULT_VALUE)); } let needle = postcard::to_stdvec(value).unwrap(); let mut prefix = needle.clone(); prefix.truncate(32); - if let Some(cand) = self.index.get(T::NAME)?.get(&prefix).copied() { - let cand_val = self.storage.get(T::NAME)?.get(&cand)?; - if *cand_val == needle { - return Some(Key::from_raw(cand)); + + if let Some(cand) = self + .index + .get(T::NAME) + .and_then(|s| s.get(&prefix)) + .copied() + { + if let Some(cand_val) = self.storage.get(T::NAME).and_then(|s| s.get(&cand)) { + if *cand_val == needle { + return Ok(Some(Key::from_raw(cand))); + } } } - None + Ok(None) } } diff --git a/fuel-compression/src/registry/mod.rs b/fuel-compression/src/registry/mod.rs index b5a1a5fb3d..bd6314f082 100644 --- a/fuel-compression/src/registry/mod.rs +++ b/fuel-compression/src/registry/mod.rs @@ -139,10 +139,10 @@ macro_rules! tables { } )* - pub fn next_keys(reg: &mut R) -> KeyPerTable { - KeyPerTable { - $( $name: reg.next_key(), )* - } + pub fn next_keys(reg: &mut R) -> anyhow::Result { + Ok(KeyPerTable { + $( $name: reg.next_key()?, )* + }) } /// Used to add together keys and counts to deterimine possible overwrite range @@ -182,10 +182,11 @@ macro_rules! tables { } /// Apply changes to the registry db - pub fn apply_to_registry(&self, reg: &mut R) { + pub fn apply_to_registry(&self, reg: &mut R) -> anyhow::Result<()> { $( - reg.batch_write(self.$name.start_key, self.$name.values.clone()); + reg.batch_write(self.$name.start_key, self.$name.values.clone())?; )* + Ok(()) } } diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compact.rs index f342df29af..7940baef6a 100644 --- a/fuel-derive/src/compact.rs +++ b/fuel-derive/src/compact.rs @@ -107,7 +107,7 @@ fn construct_compact( FieldAttrs::Skip => quote! {}, FieldAttrs::Normal => { quote! { - let #cname = <#ty as Compactable>::compact(&#binding, ctx); + let #cname = <#ty as Compactable>::compact(&#binding, ctx)?; } } FieldAttrs::Registry(registry) => { @@ -120,7 +120,7 @@ fn construct_compact( quote! { let #cname: #cty = ctx.to_key( ::Type::from(#binding.clone()) - ); + )?; } } } @@ -175,7 +175,7 @@ fn construct_decompact( }, FieldAttrs::Normal => { quote! { - let #cname = <#ty as Compactable>::decompact(#binding, reg); + let #cname = <#ty as Compactable>::decompact(#binding, reg)?; } } FieldAttrs::Registry(registry) => { @@ -183,7 +183,7 @@ fn construct_decompact( quote! { let raw: ::Type = reg.read( #binding - ); + )?; let #cname = raw.into(); } } @@ -354,12 +354,12 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { match self { #count_per_variant } } - fn compact(&self, ctx: &mut CompactionContext) -> Self::Compact { - match self { #construct_per_variant } + fn compact(&self, ctx: &mut CompactionContext) -> anyhow::Result { + Ok(match self { #construct_per_variant }) } - fn decompact(compact: Self::Compact, reg: &R) -> Self { - match compact { #decompact_per_variant } + fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result { + Ok(match compact { #decompact_per_variant }) } } }); diff --git a/fuel-tx/Cargo.toml b/fuel-tx/Cargo.toml index e79520db8f..eec8788442 100644 --- a/fuel-tx/Cargo.toml +++ b/fuel-tx/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } description = "FuelVM transaction." [dependencies] +anyhow = { workspace = true, optional = true } bitflags = { workspace = true } derivative = { version = "2.2.0", default-features = false, features = ["use_core"], optional = true } derive_more = { version = "0.99", default-features = false, features = ["display"] } @@ -54,4 +55,4 @@ std = ["alloc", "fuel-asm/std", "fuel-crypto/std", "fuel-merkle/std", "fuel-type alloc = ["hashbrown", "fuel-types/alloc", "itertools/use_alloc", "derivative", "fuel-merkle", "strum", "strum_macros"] # serde is requiring alloc because its mandatory for serde_json. to avoid adding a new feature only for serde_json, we just require `alloc` here since as of the moment we don't have a use case of serde without alloc. serde = ["alloc", "dep:serde", "fuel-asm/serde", "fuel-crypto/serde", "fuel-types/serde", "fuel-merkle/serde", "serde_json", "hashbrown/serde", "bitflags/serde"] -da-compression = ["serde", "fuel-compression", "fuel-types/da-compression"] +da-compression = ["anyhow", "serde", "fuel-compression", "fuel-types/da-compression"] diff --git a/fuel-tx/src/transaction/types/input.rs b/fuel-tx/src/transaction/types/input.rs index db45dd4e38..640b1a2f09 100644 --- a/fuel-tx/src/transaction/types/input.rs +++ b/fuel-tx/src/transaction/types/input.rs @@ -116,12 +116,15 @@ where fn compact( &self, _: &mut fuel_compression::CompactionContext, - ) -> Self::Compact { - () + ) -> anyhow::Result { + Ok(()) } - fn decompact(_: Self::Compact, _: &R) -> Self { - Self(Default::default()) + fn decompact( + _: Self::Compact, + _: &R, + ) -> anyhow::Result { + Ok(Self(Default::default())) } } diff --git a/fuel-types/Cargo.toml b/fuel-types/Cargo.toml index 5de786139c..746cb5e58e 100644 --- a/fuel-types/Cargo.toml +++ b/fuel-types/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } description = "Atomic types of the FuelVM." [dependencies] +anyhow = { workspace = true, optional = true } fuel-compression = { workspace = true } fuel-derive = { workspace = true } hex = { version = "0.4", default-features = false, optional = true } @@ -32,7 +33,7 @@ typescript = ["wasm-bindgen"] alloc = ["hex/alloc"] random = ["rand"] serde = ["dep:serde", "alloc"] -da-compression = ["serde"] +da-compression = ["anyhow", "serde"] std = ["alloc", "serde?/std", "hex?/std"] unsafe = [] diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index 8448b09cf3..5b5dfebe84 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -11,7 +11,7 @@ repository = { workspace = true } description = "FuelVM interpreter." [dependencies] -anyhow = { version = "1.0", optional = true } +anyhow = { workspace = true, optional = true } async-trait = "0.1" backtrace = { version = "0.3", optional = true } # requires debug symbols to work bitflags = { workspace = true } @@ -65,6 +65,6 @@ profile-gas = ["profile-any"] profile-coverage = ["profile-any"] profile-any = ["dyn-clone"] # All profiling features should depend on this random = ["fuel-crypto/random", "fuel-types/random", "fuel-tx/random", "rand"] -da-compression = ["fuel-compression", "fuel-tx/da-compression"] +da-compression = ["dep:anyhow", "fuel-compression", "fuel-tx/da-compression"] serde = ["dep:serde", "hashbrown/serde", "fuel-asm/serde", "fuel-types/serde", "fuel-tx/serde", "fuel-merkle/serde", "backtrace?/serde"] test-helpers = ["fuel-tx/builder", "alloc", "random", "dep:anyhow", "fuel-crypto/test-helpers", "fuel-compression/test-helpers"] From 6ad66c3314070c642481e515f5f23e022a4b6d79 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 12 Feb 2024 18:48:07 +0200 Subject: [PATCH 07/77] Update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 56869fd259..9c77b1bf19 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ and [the Sway compiler](https://github.com/FuelLabs/sway/). | Crate | Version | Description | |--------------|-----------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | fuel-asm | [![crates.io](https://img.shields.io/crates/v/fuel-asm)](https://crates.io/crates/fuel-asm) | Contains the FuelVM instruction set - opcodes used by the Sway and VM. | +| fuel-compression | [![crates.io](https://img.shields.io/crates/v/fuel-compression)](https://crates.io/crates/fuel-compression) | DA-layer compression of Fuel transaction types | | fuel-crypto | [![crates.io](https://img.shields.io/crates/v/fuel-crypto)](https://crates.io/crates/fuel-crypto) | Cryptographic primitives used across Fuel Rust based projects. | | fuel-merkle | [![crates.io](https://img.shields.io/crates/v/fuel-merkle)](https://crates.io/crates/fuel-merkle) | Implementations of the Merkle Tree used by the `fuel-core` to fulfill fraud proofs requirements, and `fuel-tx` to validate transaction validity. | | fuel-storage | [![crates.io](https://img.shields.io/crates/v/fuel-storage)](https://crates.io/crates/fuel-storage) | Storage abstraction is used to connect FuelVM, `fuel-merkle`, and `fuel-core` together without direct access. | From cc1b7a8d8b0ba75f440e5e8230d050bc6d714c29 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 6 Mar 2024 10:05:36 +0100 Subject: [PATCH 08/77] Fix wording Co-authored-by: Brandon Vrooman --- fuel-compression/src/compaction.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index d2d64e54a0..77a8bf06c2 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -176,7 +176,7 @@ fn try_map_array Result>( tmp[i] = MaybeUninit::new(f(v)?); } - // SAFETY: Everything element is initialized. + // SAFETY: Every element is initialized. Ok(tmp.map(|v| unsafe { v.assume_init() })) } From 8f60962122e7d56aa0ac528cc9b2f0b409e51c95 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 6 Mar 2024 10:07:09 +0100 Subject: [PATCH 09/77] Combine some bounds Co-authored-by: Brandon Vrooman --- fuel-compression/src/compaction.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 77a8bf06c2..77488558bd 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -72,8 +72,7 @@ impl<'a, R: RegistryDb> CompactionContext<'a, R> { /// If necessary, store the value in the changeset and allocate a new key. pub fn to_key(&mut self, value: T::Type) -> anyhow::Result> where - KeyPerTable: access::AccessCopy>, - KeyPerTable: access::AccessMut>, + KeyPerTable: access::AccessCopy> + access::AccessMut> ChangesPerTable: access::AccessRef> + access::AccessMut>, { From ac811f5a0d44f7ffef9626ba6a0acc51d2902274 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 17 Jul 2024 02:49:36 +0300 Subject: [PATCH 10/77] Fix issues after merge --- fuel-compression/Cargo.toml | 2 +- fuel-compression/src/compaction.rs | 57 ++++++++++------------ fuel-compression/src/registry/in_memory.rs | 11 +++-- fuel-compression/src/registry/mod.rs | 39 +++++++++------ 4 files changed, 59 insertions(+), 50 deletions(-) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index 8c1bd8791c..a621bb3b52 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -20,7 +20,7 @@ serde-big-array = "0.5" [dev-dependencies] bincode = "1.3" fuel-asm = { workspace = true } -fuel-compression = { path = "." } # Self-dependency needed by test for macros +fuel-compression = { path = ".", features = ["test-helpers"]} # Self-dependency needed by test for macros fuel-tx = { workspace = true } fuel-types = { workspace = true, features = ["serde"] } diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 77488558bd..0cf957e368 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -72,7 +72,7 @@ impl<'a, R: RegistryDb> CompactionContext<'a, R> { /// If necessary, store the value in the changeset and allocate a new key. pub fn to_key(&mut self, value: T::Type) -> anyhow::Result> where - KeyPerTable: access::AccessCopy> + access::AccessMut> + KeyPerTable: access::AccessCopy> + access::AccessMut>, ChangesPerTable: access::AccessRef> + access::AccessMut>, { @@ -262,16 +262,14 @@ impl Compactable for PhantomData { #[cfg(test)] mod tests { - use crate::{ - registry::{ - in_memory::InMemoryRegistry, - tables, - CountPerTable, - }, + use fuel_compression::{ + tables, + Compactable, + CompactionContext, + CountPerTable, Key, RegistryDb, }; - use fuel_compression::Compactable as _; // Hack for derive use fuel_derive::Compact; use fuel_types::{ Address, @@ -282,11 +280,6 @@ mod tests { Serialize, }; - use super::{ - Compactable, - CompactionContext, - }; - #[derive(Debug, Clone, PartialEq)] struct ManualExample { a: Address, @@ -304,26 +297,26 @@ mod tests { impl Compactable for ManualExample { type Compact = ManualExampleCompact; - fn count(&self) -> crate::registry::CountPerTable { - CountPerTable { - Address: 2, - ..Default::default() - } + fn count(&self) -> CountPerTable { + CountPerTable::Address(2) } fn compact( &self, ctx: &mut CompactionContext, - ) -> Self::Compact { - let a = ctx.to_key::(*self.a); - let b = ctx.to_key::(*self.b); - ManualExampleCompact { a, b, c: self.c } + ) -> anyhow::Result { + let a = ctx.to_key::(*self.a)?; + let b = ctx.to_key::(*self.b)?; + Ok(ManualExampleCompact { a, b, c: self.c }) } - fn decompact(compact: Self::Compact, reg: &R) -> Self { - let a = Address::from(reg.read::(compact.a)); - let b = Address::from(reg.read::(compact.b)); - Self { a, b, c: compact.c } + fn decompact( + compact: Self::Compact, + reg: &R, + ) -> anyhow::Result { + let a = Address::from(reg.read::(compact.a)?); + let b = Address::from(reg.read::(compact.b)?); + Ok(Self { a, b, c: compact.c }) } } @@ -362,9 +355,10 @@ mod tests { b: Address::from([2u8; 32]), c: 3, }; - let mut registry = InMemoryRegistry::default(); - let (compacted, _) = CompactionContext::run(&mut registry, target.clone()); - let decompacted = ManualExample::decompact(compacted, ®istry); + let mut registry = fuel_compression::InMemoryRegistry::default(); + let (compacted, _) = + CompactionContext::run(&mut registry, target.clone()).unwrap(); + let decompacted = ManualExample::decompact(compacted, ®istry).unwrap(); assert_eq!(target, decompacted); let target = AutomaticExample { @@ -374,8 +368,9 @@ mod tests { }; let mut registry = fuel_compression::InMemoryRegistry::default(); let (compacted, _) = - fuel_compression::CompactionContext::run(&mut registry, target.clone()); - let decompacted = AutomaticExample::decompact(compacted, ®istry); + fuel_compression::CompactionContext::run(&mut registry, target.clone()) + .unwrap(); + let decompacted = AutomaticExample::decompact(compacted, ®istry).unwrap(); assert_eq!(target, decompacted); } } diff --git a/fuel-compression/src/registry/in_memory.rs b/fuel-compression/src/registry/in_memory.rs index 344b573af7..ba740f52a5 100644 --- a/fuel-compression/src/registry/in_memory.rs +++ b/fuel-compression/src/registry/in_memory.rs @@ -100,7 +100,8 @@ mod tests { // Empty assert_eq!( - reg.read(Key::::try_from(100).unwrap()), + reg.read(Key::::try_from(100).unwrap()) + .unwrap(), [0; 32] ); @@ -108,17 +109,19 @@ mod tests { reg.batch_write( Key::::from_raw(RawKey::try_from(100u32).unwrap()), vec![[1; 32], [2; 32]], - ); + ) + .unwrap(); // Read assert_eq!( - reg.read(Key::::try_from(100).unwrap()), + reg.read(Key::::try_from(100).unwrap()) + .unwrap(), [1; 32] ); // Index assert_eq!( - reg.index_lookup(&[1; 32]), + reg.index_lookup(&[1; 32]).unwrap(), Some(Key::::try_from(100).unwrap()) ); } diff --git a/fuel-compression/src/registry/mod.rs b/fuel-compression/src/registry/mod.rs index bd6314f082..e24743c67d 100644 --- a/fuel-compression/src/registry/mod.rs +++ b/fuel-compression/src/registry/mod.rs @@ -225,11 +225,12 @@ mod tests { // Empty assert_eq!( - reg.read(Key::::try_from(100).unwrap()), + reg.read(Key::::try_from(100).unwrap()) + .unwrap(), [0; 32] ); assert_eq!( - reg.index_lookup(&*AssetId::from([1; 32])), + reg.index_lookup(&*AssetId::from([1; 32])).unwrap(), None::> ); @@ -237,17 +238,21 @@ mod tests { reg.batch_write( Key::::from_raw(RawKey::try_from(100u32).unwrap()), vec![[1; 32], [2; 32]], - ); + ) + .unwrap(); assert_eq!( - reg.read(Key::::try_from(100).unwrap()), + reg.read(Key::::try_from(100).unwrap()) + .unwrap(), [1; 32] ); assert_eq!( - reg.read(Key::::try_from(101).unwrap()), + reg.read(Key::::try_from(101).unwrap()) + .unwrap(), [2; 32] ); assert_eq!( - reg.read(Key::::try_from(102).unwrap()), + reg.read(Key::::try_from(102).unwrap()) + .unwrap(), [0; 32] ); @@ -255,13 +260,16 @@ mod tests { reg.batch_write( Key::::from_raw(RawKey::try_from(99u32).unwrap()), vec![[10; 32], [11; 32]], - ); + ) + .unwrap(); assert_eq!( - reg.read(Key::::try_from(99).unwrap()), + reg.read(Key::::try_from(99).unwrap()) + .unwrap(), [10; 32] ); assert_eq!( - reg.read(Key::::try_from(100).unwrap()), + reg.read(Key::::try_from(100).unwrap()) + .unwrap(), [11; 32] ); @@ -269,25 +277,28 @@ mod tests { reg.batch_write( Key::::from_raw(RawKey::MAX_WRITABLE), vec![[3; 32], [4; 32]], - ); + ) + .unwrap(); assert_eq!( - reg.read(Key::::from_raw(RawKey::MAX_WRITABLE)), + reg.read(Key::::from_raw(RawKey::MAX_WRITABLE)) + .unwrap(), [3; 32] ); assert_eq!( - reg.read(Key::::from_raw(RawKey::ZERO)), + reg.read(Key::::from_raw(RawKey::ZERO)) + .unwrap(), [4; 32] ); assert_eq!( - reg.index_lookup(&*AssetId::from([3; 32])), + reg.index_lookup(&*AssetId::from([3; 32])).unwrap(), Some(Key::::from_raw(RawKey::MAX_WRITABLE)) ); assert_eq!( - reg.index_lookup(&*AssetId::from([4; 32])), + reg.index_lookup(&*AssetId::from([4; 32])).unwrap(), Some(Key::::from_raw(RawKey::ZERO)) ); } From 8ed067266f0f37c4cd7656a0ce8a7872076befee Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 17 Jul 2024 05:08:44 +0300 Subject: [PATCH 11/77] Fix compression for latest types --- fuel-compression/src/compaction.rs | 4 +- .../src/registry/block_section.rs | 2 +- fuel-compression/src/registry/key.rs | 4 +- fuel-derive/src/compact.rs | 150 ++++++++++++++++-- .../types/chargeable_transaction.rs | 5 +- fuel-tx/src/transaction/types/input/coin.rs | 2 +- .../src/transaction/types/input/message.rs | 2 +- fuel-tx/src/transaction/types/upgrade.rs | 2 + fuel-tx/src/transaction/types/upload.rs | 1 + fuel-tx/src/transaction/types/utxo_id.rs | 23 --- 10 files changed, 154 insertions(+), 41 deletions(-) diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 0cf957e368..b7c61392a0 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -203,9 +203,7 @@ where } fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result { - Ok(try_map_array(compact.0, |v: T::Compact| { - T::decompact(v, reg) - })?) + try_map_array(compact.0, |v: T::Compact| T::decompact(v, reg)) } } diff --git a/fuel-compression/src/registry/block_section.rs b/fuel-compression/src/registry/block_section.rs index 2e4e73a625..a56a996b4a 100644 --- a/fuel-compression/src/registry/block_section.rs +++ b/fuel-compression/src/registry/block_section.rs @@ -110,7 +110,7 @@ impl<'de, T: Table + Deserialize<'de>> serde::de::Visitor<'de> for WriteTo { )?; if values.is_empty() { - let _: () = seq.next_element()?.ok_or(serde::de::Error::invalid_length( + seq.next_element()?.ok_or(serde::de::Error::invalid_length( 1, &"WriteTo<_> with 2 elements", ))?; diff --git a/fuel-compression/src/registry/key.rs b/fuel-compression/src/registry/key.rs index 9da4df9528..88ee5072af 100644 --- a/fuel-compression/src/registry/key.rs +++ b/fuel-compression/src/registry/key.rs @@ -24,6 +24,7 @@ impl RawKey { } /// Wraps around just below max/default value. + #[allow(clippy::cast_possible_truncation)] pub fn add_u32(self, rhs: u32) -> Self { let lhs = self.as_u32() as u64; let rhs = rhs as u64; @@ -87,12 +88,13 @@ impl TryFrom for RawKey { /// Typed key to a registry table entry. /// The last key (all bits set) is reserved for the default value and cannot be written /// to. +#[allow(clippy::derived_hash_with_manual_eq)] // PhantomData requires this #[derive(Eq, Hash, Serialize, Deserialize)] #[serde(transparent)] pub struct Key(RawKey, PhantomData); impl Clone for Key { fn clone(&self) -> Self { - Self(self.0, PhantomData) + *self } } impl Copy for Key {} diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compact.rs index 7940baef6a..018c88ef94 100644 --- a/fuel-derive/src/compact.rs +++ b/fuel-derive/src/compact.rs @@ -1,13 +1,97 @@ -use proc_macro2::TokenStream as TokenStream2; +use proc_macro2::{ + Span, + TokenStream as TokenStream2, + TokenTree as TokenTree2, +}; use quote::{ format_ident, quote, }; use regex::Regex; +use syn::parse::{ + Parse, + ParseStream, +}; const ATTR: &str = "da_compress"; +/// Structure (struct or enum) attributes +#[derive(Debug)] +pub enum StructureAttrs { + /// Insert bounds for a generic type + Bound(Vec), + /// Discard generic parameter + Discard(Vec), +} +impl Parse for StructureAttrs { + fn parse(input: ParseStream) -> syn::Result { + if let Ok(ml) = input.parse::() { + if ml.path.segments.len() == 1 { + match ml.path.segments[0].ident.to_string().as_str() { + "bound" => { + let mut bound = Vec::new(); + for item in ml.tokens { + match item { + TokenTree2::Ident(ident) => { + bound.push(ident.to_string()); + } + other => { + return Err(syn::Error::new_spanned( + other, + "Expected generic (type) name", + )) + } + } + } + return Ok(Self::Bound(bound)); + } + "discard" => { + let mut discard = Vec::new(); + for item in ml.tokens { + match item { + TokenTree2::Ident(ident) => { + discard.push(ident.to_string()); + } + other => { + return Err(syn::Error::new_spanned( + other, + "Expected generic (type) name", + )) + } + } + } + return Ok(Self::Discard(discard)); + } + _ => {} + } + } + } + Err(syn::Error::new_spanned( + input.parse::()?, + "Expected `bound` or `discard`", + )) + } +} +impl StructureAttrs { + pub fn parse(attrs: &[syn::Attribute]) -> syn::Result> { + let mut result = Vec::new(); + for attr in attrs { + if attr.style != syn::AttrStyle::Outer { + continue; + } + + if let syn::Meta::List(ml) = &attr.meta { + if ml.path.segments.len() == 1 && ml.path.segments[0].ident == ATTR { + result.push(syn::parse2::(ml.tokens.clone())?); + } + } + } + + Ok(result) + } +} + /// Field attributes pub enum FieldAttrs { /// Skipped when compacting, and must be reconstructed when decompacting. @@ -280,15 +364,61 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { s.add_bounds(synstructure::AddBounds::None) .underscore_const(true); + let s_attrs = match StructureAttrs::parse(&s.ast().attrs) { + Ok(v) => v, + Err(e) => return e.to_compile_error(), + }; + let name = &s.ast().ident; let compact_name = format_ident!("Compact{}", name); - let g = s.ast().generics.clone(); - let w = g.where_clause.clone(); + let mut g = s.ast().generics.clone(); + let mut w_structure = g.where_clause.take(); + let mut w_impl = w_structure.clone(); + for item in &s_attrs { + match item { + StructureAttrs::Bound(bound) => { + if w_structure.is_none() { + w_structure = Some(syn::WhereClause { + where_token: syn::Token![where](proc_macro2::Span::call_site()), + predicates: Default::default(), + }); + w_impl = Some(syn::WhereClause { + where_token: syn::Token![where](proc_macro2::Span::call_site()), + predicates: Default::default(), + }); + } + for p in bound { + let id = syn::Ident::new(p, Span::call_site()); + w_structure + .as_mut() + .unwrap() + .predicates + .push(syn::parse_quote! { #id: ::fuel_compression::Compactable }); + w_impl.as_mut().unwrap().predicates.push( + syn::parse_quote! { for<'de> #id: ::fuel_compression::Compactable + serde::Serialize + serde::Deserialize<'de> + Clone }, + ); + } + } + StructureAttrs::Discard(discard) => { + g.params = g + .params + .into_pairs() + .filter(|pair| match pair.value() { + syn::GenericParam::Type(t) => { + !discard.contains(&t.ident.to_string()) + } + _ => true, + }) + .collect(); + } + } + } + let def = match &s.ast().data { syn::Data::Struct(v) => { let variant: &synstructure::VariantInfo = &s.variants()[0]; - let defs = field_defs(&variant.ast().fields); + let defs = field_defs(variant.ast().fields); let semi = match v.fields { syn::Fields::Named(_) => quote! {}, syn::Fields::Unnamed(_) => quote! {;}, @@ -297,7 +427,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { quote! { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[doc = concat!("Compacted version of `", stringify!(#name), "`.")] - pub struct #compact_name #g #w #defs #semi + pub struct #compact_name #g #w_structure #defs #semi } } syn::Data::Enum(_) => { @@ -306,7 +436,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { .iter() .map(|variant| { let vname = variant.ast().ident.clone(); - let defs = field_defs(&variant.ast().fields); + let defs = field_defs(variant.ast().fields); quote! { #vname #defs, } @@ -316,13 +446,13 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { quote! { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[doc = concat!("Compacted version of `", stringify!(#name), "`.")] - pub enum #compact_name #g #w { #variant_defs } + pub enum #compact_name #g #w_structure { #variant_defs } } } syn::Data::Union(_) => panic!("unions are not supported"), }; - let count_per_variant = s.each_variant(|variant| sum_counts(variant)); + let count_per_variant = s.each_variant(sum_counts); let construct_per_variant = s.each_variant(|variant| { let vname = variant.ast().ident.clone(); let construct = match &s.ast().data { @@ -347,7 +477,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { let impls = s.gen_impl(quote! { use ::fuel_compression::{RegistryDb, tables, Table, Key, Compactable, CountPerTable, CompactionContext}; - gen impl Compactable for @Self { + gen impl Compactable for @Self #w_impl { type Compact = #compact_name #g; fn count(&self) -> CountPerTable { @@ -368,7 +498,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { #impls }; - let _ = std::fs::write(format!("/tmp/derive/{}.rs", name), &rs.to_string()); + let _ = std::fs::write(format!("/tmp/derive/{}.rs", name), rs.to_string()); rs } diff --git a/fuel-tx/src/transaction/types/chargeable_transaction.rs b/fuel-tx/src/transaction/types/chargeable_transaction.rs index b4ee2c48b2..c44b70c171 100644 --- a/fuel-tx/src/transaction/types/chargeable_transaction.rs +++ b/fuel-tx/src/transaction/types/chargeable_transaction.rs @@ -43,9 +43,12 @@ pub struct ChargeableMetadata { } #[derive(Clone, Derivative)] +#[derivative(Eq, PartialEq, Hash, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", da_compress(bound(Body)))] +#[cfg_attr(feature = "da-compression", da_compress(discard(MetadataBody)))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[derivative(Eq, PartialEq, Hash, Debug)] pub struct ChargeableTransaction { pub(crate) body: Body, pub(crate) policies: Policies, diff --git a/fuel-tx/src/transaction/types/input/coin.rs b/fuel-tx/src/transaction/types/input/coin.rs index 183ac772ab..e75cc7f70c 100644 --- a/fuel-tx/src/transaction/types/input/coin.rs +++ b/fuel-tx/src/transaction/types/input/coin.rs @@ -34,7 +34,7 @@ mod private { /// Specifies the coin based on the usage context. See [`Coin`]. #[cfg(feature = "da-compression")] pub trait CoinSpecification: private::Seal { - type Witness: AsField + Compactable + Clone; + type Witness: AsField + Compactable + Clone; type Predicate: AsField> + Compactable + Clone; type PredicateData: AsField> + Compactable + Clone; type PredicateGasUsed: AsField + Compactable + Clone; diff --git a/fuel-tx/src/transaction/types/input/message.rs b/fuel-tx/src/transaction/types/input/message.rs index c316529f31..6b5dbc6f22 100644 --- a/fuel-tx/src/transaction/types/input/message.rs +++ b/fuel-tx/src/transaction/types/input/message.rs @@ -55,7 +55,7 @@ pub trait MessageSpecification: private::Seal { + serde::Serialize + for<'a> serde::Deserialize<'a> + Default; - type Witness: AsField + type Witness: AsField + Compactable + Clone + serde::Serialize diff --git a/fuel-tx/src/transaction/types/upgrade.rs b/fuel-tx/src/transaction/types/upgrade.rs index 2ed6a97c65..5dc445b2e9 100644 --- a/fuel-tx/src/transaction/types/upgrade.rs +++ b/fuel-tx/src/transaction/types/upgrade.rs @@ -96,6 +96,7 @@ impl UpgradeMetadata { /// transaction. #[derive(Copy, Clone, Derivative, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[derivative(Eq, PartialEq, Hash, Debug)] pub enum UpgradePurpose { @@ -122,6 +123,7 @@ pub enum UpgradePurpose { /// The body of the [`Upgrade`] transaction. #[derive(Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Upgrade)] #[derivative(Eq, PartialEq, Hash, Debug)] diff --git a/fuel-tx/src/transaction/types/upload.rs b/fuel-tx/src/transaction/types/upload.rs index 9893b2ad62..067b9eb55d 100644 --- a/fuel-tx/src/transaction/types/upload.rs +++ b/fuel-tx/src/transaction/types/upload.rs @@ -39,6 +39,7 @@ pub struct UploadMetadata; /// The body of the [`Upload`] transaction. #[derive(Clone, Default, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Upload)] #[derivative(Eq, PartialEq, Hash, Debug)] diff --git a/fuel-tx/src/transaction/types/utxo_id.rs b/fuel-tx/src/transaction/types/utxo_id.rs index 391cb7541d..3af2cb2e58 100644 --- a/fuel-tx/src/transaction/types/utxo_id.rs +++ b/fuel-tx/src/transaction/types/utxo_id.rs @@ -29,29 +29,6 @@ pub struct UtxoId { output_index: u16, } -// const _: () = { -// impl fuel_compression::Compactable for UtxoId { -// type Compact = (); - -// fn count(&self) -> fuel_compression::CountPerTable { -// todo!() -// } - -// fn compact(&self, ctx: &mut fuel_compression::CompactionContext) -> -// Self::Compact where -// R: fuel_compression::db::RegistryRead + -// fuel_compression::db::RegistryWrite + fuel_compression::db::RegistryIndex { -// todo!() -// } - -// fn decompact(compact: Self::Compact, reg: &R) -> Self -// where -// R: fuel_compression::db::RegistryRead { -// todo!() -// } -// } -// }; - impl UtxoId { pub const LEN: usize = TxId::LEN + 8; From f04df709b5db7a5a80de5f41b65e2f6be53e22fd Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 17 Jul 2024 14:32:40 +0300 Subject: [PATCH 12/77] Keep used gas price --- fuel-tx/src/transaction/types/input/message.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/fuel-tx/src/transaction/types/input/message.rs b/fuel-tx/src/transaction/types/input/message.rs index 6b5dbc6f22..91daf59720 100644 --- a/fuel-tx/src/transaction/types/input/message.rs +++ b/fuel-tx/src/transaction/types/input/message.rs @@ -194,7 +194,6 @@ where #[derivative(Debug(format_with = "fmt_as_field"))] pub witness_index: Specification::Witness, #[derivative(Debug(format_with = "fmt_as_field"))] - #[cfg_attr(feature = "da-compression", da_compress(skip))] pub predicate_gas_used: Specification::PredicateGasUsed, #[derivative(Debug(format_with = "fmt_as_field"))] pub data: Specification::Data, From 141efe53bbd9d4e49074ef931cfdb5d6131b9a37 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 17 Jul 2024 14:33:30 +0300 Subject: [PATCH 13/77] Cargo sort --- fuel-compression/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index a621bb3b52..027dfdd442 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -20,7 +20,7 @@ serde-big-array = "0.5" [dev-dependencies] bincode = "1.3" fuel-asm = { workspace = true } -fuel-compression = { path = ".", features = ["test-helpers"]} # Self-dependency needed by test for macros +fuel-compression = { path = ".", features = ["test-helpers"] } # Self-dependency needed by test for macros fuel-tx = { workspace = true } fuel-types = { workspace = true, features = ["serde"] } From da7819fc371307ef601408df12186c167a7a716f Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 18 Jul 2024 13:24:02 +0300 Subject: [PATCH 14/77] Introduce ContractId table to da compression registry --- fuel-compression/src/registry/block_section.rs | 11 +++++++++-- fuel-compression/src/registry/mod.rs | 1 + fuel-tx/src/transaction/types/output.rs | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fuel-compression/src/registry/block_section.rs b/fuel-compression/src/registry/block_section.rs index a56a996b4a..6bf912457a 100644 --- a/fuel-compression/src/registry/block_section.rs +++ b/fuel-compression/src/registry/block_section.rs @@ -141,7 +141,10 @@ mod tests { use super::*; use bincode::Options; use fuel_asm::op; - use fuel_tx::AssetId; + use fuel_tx::{ + AssetId, + ContractId, + }; use fuel_types::Address; #[test] @@ -155,7 +158,11 @@ mod tests { }, Address: WriteTo { start_key: Key::ZERO, - values: vec![*Address::from([0xc0; 32])], + values: vec![*Address::from([0xb0; 32])], + }, + ContractId: WriteTo { + start_key: Key::ZERO, + values: vec![*ContractId::from([0xc0; 32])], }, ScriptCode: WriteTo { start_key: Key::ZERO, diff --git a/fuel-compression/src/registry/mod.rs b/fuel-compression/src/registry/mod.rs index e24743c67d..9185caa252 100644 --- a/fuel-compression/src/registry/mod.rs +++ b/fuel-compression/src/registry/mod.rs @@ -208,6 +208,7 @@ macro_rules! tables { tables!( AssetId: [u8; 32], Address: [u8; 32], + ContractId: [u8; 32], ScriptCode: Vec, Witness: Vec, ); diff --git a/fuel-tx/src/transaction/types/output.rs b/fuel-tx/src/transaction/types/output.rs index cde776fe8d..f4d3308069 100644 --- a/fuel-tx/src/transaction/types/output.rs +++ b/fuel-tx/src/transaction/types/output.rs @@ -53,6 +53,7 @@ pub enum Output { }, ContractCreated { + #[cfg_attr(feature = "da-compression", da_compress(registry = "ContractId"))] contract_id: ContractId, state_root: Bytes32, }, From 42e86341196654f2c43a2464390e8ac3748d24c7 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 18 Jul 2024 14:00:33 +0300 Subject: [PATCH 15/77] Reference table types directly in #[da_compress(registry = _)] --- fuel-compression/src/compaction.rs | 4 +- fuel-compression/src/registry/mod.rs | 6 +++ fuel-derive/src/compact.rs | 41 ++++++++++--------- fuel-tx/src/transaction/types/input/coin.rs | 4 +- .../src/transaction/types/input/message.rs | 4 +- fuel-tx/src/transaction/types/mint.rs | 2 +- fuel-tx/src/transaction/types/output.rs | 10 ++--- fuel-tx/src/transaction/types/script.rs | 2 +- fuel-tx/src/transaction/types/witness.rs | 2 +- 9 files changed, 42 insertions(+), 33 deletions(-) diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index b7c61392a0..725389ccb7 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -320,9 +320,9 @@ mod tests { #[derive(Debug, Clone, PartialEq, Compact)] struct AutomaticExample { - #[da_compress(registry = "AssetId")] + #[da_compress(registry = ::fuel_compression::tables::AssetId)] a: AssetId, - #[da_compress(registry = "AssetId")] + #[da_compress(registry = ::fuel_compression::tables::AssetId)] b: AssetId, c: u32, } diff --git a/fuel-compression/src/registry/mod.rs b/fuel-compression/src/registry/mod.rs index 9185caa252..75b30aabbf 100644 --- a/fuel-compression/src/registry/mod.rs +++ b/fuel-compression/src/registry/mod.rs @@ -23,6 +23,9 @@ pub trait Table: _private::Seal { /// Unique name of the table const NAME: &'static str; + /// A `CountPerTable` for this table + fn count(n: usize) -> CountPerTable; + /// The type stored in the table type Type: PartialEq + Default + Serialize + for<'de> Deserialize<'de>; } @@ -55,6 +58,9 @@ macro_rules! tables { impl super::_private::Seal for $name {} impl super::Table for $name { const NAME: &'static str = stringify!($name); + fn count(n: usize) -> super::CountPerTable { + super::CountPerTable::$name(n) + } type Type = $ty; } )* diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compact.rs index 018c88ef94..1df4f78091 100644 --- a/fuel-derive/src/compact.rs +++ b/fuel-derive/src/compact.rs @@ -8,7 +8,6 @@ use quote::{ quote, }; -use regex::Regex; use syn::parse::{ Parse, ParseStream, @@ -99,11 +98,11 @@ pub enum FieldAttrs { /// Compacted recursively. Normal, /// This value is compacted into a registry lookup. - Registry(String), + Registry(syn::Path), } impl FieldAttrs { pub fn parse(attrs: &[syn::Attribute]) -> Self { - let re_registry = Regex::new(r#"^registry\s*=\s*"([a-zA-Z_]+)"$"#).unwrap(); + let registry_path = syn::parse2::(quote! {registry}).unwrap(); let mut result = Self::Normal; for attr in attrs { @@ -117,14 +116,22 @@ impl FieldAttrs { panic!("Duplicate attribute: {}", ml.tokens); } - let attr_contents = ml.tokens.to_string(); - if attr_contents == "skip" { - result = Self::Skip; - } else if let Some(m) = re_registry.captures(&attr_contents) { - result = Self::Registry(m.get(1).unwrap().as_str().to_owned()); - } else { - panic!("Invalid attribute: {}", ml.tokens); + if let Ok(ident) = syn::parse2::(ml.tokens.clone()) { + if ident.to_string() == "skip" { + result = Self::Skip; + continue; + } + } else if let Ok(kv) = + syn::parse2::(ml.tokens.clone()) + { + if kv.path == registry_path { + if let syn::Expr::Path(p) = kv.value { + result = Self::Registry(p.path); + continue; + } + } } + panic!("Invalid attribute: {}", ml.tokens); } } } @@ -153,9 +160,8 @@ fn field_defs(fields: &syn::Fields) -> TokenStream2 { } } FieldAttrs::Registry(registry) => { - let reg_ident = format_ident!("{}", registry); let cty = quote! { - ::fuel_compression::Key<::fuel_compression::tables::#reg_ident> + ::fuel_compression::Key<#registry> }; if let Some(fname) = field.ident.as_ref() { quote! { #fname: #cty, } @@ -195,15 +201,14 @@ fn construct_compact( } } FieldAttrs::Registry(registry) => { - let reg_ident = format_ident!("{}", registry); let cty = quote! { Key< - tables::#reg_ident + #registry > }; quote! { let #cname: #cty = ctx.to_key( - ::Type::from(#binding.clone()) + <#registry as Table>::Type::from(#binding.clone()) )?; } } @@ -263,9 +268,8 @@ fn construct_decompact( } } FieldAttrs::Registry(registry) => { - let reg_ident = format_ident!("{}", registry); quote! { - let raw: ::Type = reg.read( + let raw: <#registry as Table>::Type = reg.read( #binding )?; let #cname = raw.into(); @@ -315,9 +319,8 @@ fn sum_counts(variant: &synstructure::VariantInfo<'_>) -> TokenStream2 { quote! { <#ty as Compactable>::count(&#binding) } } FieldAttrs::Registry(registry) => { - let reg_ident = format_ident!("{}", registry); quote! { - CountPerTable::#reg_ident(1) + #registry::count(1) } } } diff --git a/fuel-tx/src/transaction/types/input/coin.rs b/fuel-tx/src/transaction/types/input/coin.rs index e75cc7f70c..ede356b924 100644 --- a/fuel-tx/src/transaction/types/input/coin.rs +++ b/fuel-tx/src/transaction/types/input/coin.rs @@ -114,10 +114,10 @@ where Specification: CoinSpecification + Clone, { pub utxo_id: UtxoId, - #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] pub owner: Address, pub amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] pub asset_id: AssetId, pub tx_pointer: TxPointer, #[derivative(Debug(format_with = "fmt_as_field"))] diff --git a/fuel-tx/src/transaction/types/input/message.rs b/fuel-tx/src/transaction/types/input/message.rs index 91daf59720..cec7f1d4eb 100644 --- a/fuel-tx/src/transaction/types/input/message.rs +++ b/fuel-tx/src/transaction/types/input/message.rs @@ -184,10 +184,10 @@ where Specification: MessageSpecification + Clone, { /// The sender from the L1 chain. - #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] pub sender: Address, /// The receiver on the `Fuel` chain. - #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] pub recipient: Address, pub amount: Word, pub nonce: Nonce, diff --git a/fuel-tx/src/transaction/types/mint.rs b/fuel-tx/src/transaction/types/mint.rs index a15c420626..26948d2cff 100644 --- a/fuel-tx/src/transaction/types/mint.rs +++ b/fuel-tx/src/transaction/types/mint.rs @@ -64,7 +64,7 @@ pub struct Mint { /// The amount of funds minted. pub(crate) mint_amount: Word, /// The asset IDs corresponding to the minted amount. - #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] pub(crate) mint_asset_id: AssetId, /// Gas Price used for current block pub(crate) gas_price: Word, diff --git a/fuel-tx/src/transaction/types/output.rs b/fuel-tx/src/transaction/types/output.rs index f4d3308069..6eea7aed53 100644 --- a/fuel-tx/src/transaction/types/output.rs +++ b/fuel-tx/src/transaction/types/output.rs @@ -25,21 +25,21 @@ pub use repr::OutputRepr; #[derive(canonical::Deserialize, canonical::Serialize)] pub enum Output { Coin { - #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] to: Address, amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] asset_id: AssetId, }, Contract(Contract), Change { - #[cfg_attr(feature = "da-compression", da_compress(registry = "Address"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] to: Address, #[cfg_attr(feature = "da-compression", da_compress(skip))] amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry = "AssetId"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] asset_id: AssetId, }, @@ -53,7 +53,7 @@ pub enum Output { }, ContractCreated { - #[cfg_attr(feature = "da-compression", da_compress(registry = "ContractId"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::ContractId))] contract_id: ContractId, state_root: Bytes32, }, diff --git a/fuel-tx/src/transaction/types/script.rs b/fuel-tx/src/transaction/types/script.rs index 32dc4b4d1d..0121bdfb42 100644 --- a/fuel-tx/src/transaction/types/script.rs +++ b/fuel-tx/src/transaction/types/script.rs @@ -55,7 +55,7 @@ pub struct ScriptBody { pub(crate) script_gas_limit: Word, pub(crate) receipts_root: Bytes32, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] - #[cfg_attr(feature = "da-compression", da_compress(registry = "ScriptCode"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::ScriptCode))] pub(crate) script: Vec, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] pub(crate) script_data: Vec, diff --git a/fuel-tx/src/transaction/types/witness.rs b/fuel-tx/src/transaction/types/witness.rs index cde5bbb25a..9d0761b0c3 100644 --- a/fuel-tx/src/transaction/types/witness.rs +++ b/fuel-tx/src/transaction/types/witness.rs @@ -30,7 +30,7 @@ use rand::{ #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] pub struct Witness { #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] - #[cfg_attr(feature = "da-compression", da_compress(registry = "Witness"))] + #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Witness))] data: Vec, } From 781fe1e7b6ff9e3da268bc319ec70330a80ce708 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 22 Jul 2024 08:07:50 +0300 Subject: [PATCH 16/77] WIP: migrate domain logic to fuel-core --- fuel-compression/Cargo.toml | 3 +- .../src/{registry => }/block_section.rs | 0 fuel-compression/src/compaction.rs | 164 ++++-------------- fuel-compression/src/{registry => }/key.rs | 0 fuel-compression/src/lib.rs | 18 +- fuel-compression/src/registry/db.rs | 25 --- fuel-compression/src/registry/in_memory.rs | 128 -------------- .../src/{registry/mod.rs => table.rs} | 63 ++++--- fuel-derive/src/compact.rs | 23 ++- fuel-tx/src/transaction/types/input.rs | 8 +- fuel-vm/Cargo.toml | 2 +- 11 files changed, 103 insertions(+), 331 deletions(-) rename fuel-compression/src/{registry => }/block_section.rs (100%) rename fuel-compression/src/{registry => }/key.rs (100%) delete mode 100644 fuel-compression/src/registry/db.rs delete mode 100644 fuel-compression/src/registry/in_memory.rs rename fuel-compression/src/{registry/mod.rs => table.rs} (83%) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index 027dfdd442..89b85636bb 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -13,12 +13,11 @@ description = "Compression and decompression of Fuel blocks for DA storage." [dependencies] anyhow = { workspace = true } fuel-derive = { workspace = true } -postcard = { version = "1.0", features = ["use-std"] } +paste = "1.0" serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" [dev-dependencies] -bincode = "1.3" fuel-asm = { workspace = true } fuel-compression = { path = ".", features = ["test-helpers"] } # Self-dependency needed by test for macros fuel-tx = { workspace = true } diff --git a/fuel-compression/src/registry/block_section.rs b/fuel-compression/src/block_section.rs similarity index 100% rename from fuel-compression/src/registry/block_section.rs rename to fuel-compression/src/block_section.rs diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 725389ccb7..781c77d9fd 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -9,100 +9,12 @@ use serde::{ }; use crate::{ - registry::{ - access::{ - self, - *, - }, - add_keys, - block_section::WriteTo, - next_keys, - ChangesPerTable, - CountPerTable, - KeyPerTable, - RegistryDb, - Table, - }, + CompactionContext, + CountPerTable, + DecompactionContext, Key, }; -/// Context for compaction, i.e. converting data to reference-based format -#[must_use] -pub struct CompactionContext<'a, R> { - /// The registry - reg: &'a mut R, - /// These are the keys where writing started - start_keys: KeyPerTable, - /// The next keys to use for each table - next_keys: KeyPerTable, - /// Keys in range next_keys..safe_keys_start - /// could be overwritten by the compaction, - /// and cannot be used for new values. - safe_keys_start: KeyPerTable, - changes: ChangesPerTable, -} -impl<'a, R: RegistryDb> CompactionContext<'a, R> { - /// Run the compaction for the given target, returning the compacted data. - /// Changes are applied to the registry, and then returned as well. - pub fn run( - reg: &'a mut R, - target: C, - ) -> anyhow::Result<(C::Compact, ChangesPerTable)> { - let start_keys = next_keys(reg)?; - let next_keys = start_keys; - let key_limits = target.count(); - let safe_keys_start = add_keys(next_keys, key_limits); - - let mut ctx = Self { - reg, - start_keys, - next_keys, - safe_keys_start, - changes: ChangesPerTable::from_start_keys(start_keys), - }; - - let compacted = target.compact(&mut ctx)?; - ctx.changes.apply_to_registry(ctx.reg)?; - Ok((compacted, ctx.changes)) - } -} - -impl<'a, R: RegistryDb> CompactionContext<'a, R> { - /// Convert a value to a key - /// If necessary, store the value in the changeset and allocate a new key. - pub fn to_key(&mut self, value: T::Type) -> anyhow::Result> - where - KeyPerTable: access::AccessCopy> + access::AccessMut>, - ChangesPerTable: - access::AccessRef> + access::AccessMut>, - { - // Check if the value is within the current changeset - if let Some(key) = - >>::get(&self.changes) - .lookup_value(&value) - { - return Ok(key); - } - - // Check if the registry contains this value already - if let Some(key) = self.reg.index_lookup::(&value)? { - let start: Key = self.start_keys.value(); - let end: Key = self.safe_keys_start.value(); - // Check if the value is in the possibly-overwritable range - if !key.is_between(start, end) { - return Ok(key); - } - } - // Allocate a new key for this - let key = >>::get_mut(&mut self.next_keys) - .take_next(); - >>::get_mut(&mut self.changes) - .values - .push(value); - Ok(key) - } -} - /// Convert data to reference-based format pub trait Compactable { /// The compacted type with references @@ -112,13 +24,13 @@ pub trait Compactable { fn count(&self) -> CountPerTable; /// Convert to compacted format - fn compact( - &self, - ctx: &mut CompactionContext, - ) -> anyhow::Result; + fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result; /// Convert from compacted format - fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result + fn decompact( + compact: Self::Compact, + ctx: &dyn DecompactionContext, + ) -> anyhow::Result where Self: Sized; } @@ -132,16 +44,16 @@ macro_rules! identity_compaction { CountPerTable::default() } - fn compact( + fn compact( &self, - _ctx: &mut CompactionContext, + _ctx: &mut dyn CompactionContext, ) -> anyhow::Result { Ok(*self) } - fn decompact( + fn decompact( compact: Self::Compact, - _reg: &R, + _ctx: &dyn DecompactionContext, ) -> anyhow::Result { Ok(compact) } @@ -193,17 +105,17 @@ where count } - fn compact( - &self, - ctx: &mut CompactionContext, - ) -> anyhow::Result { + fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result { Ok(ArrayWrapper(try_map_array(self.clone(), |v: T| { v.compact(ctx) })?)) } - fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result { - try_map_array(compact.0, |v: T::Compact| T::decompact(v, reg)) + fn decompact( + compact: Self::Compact, + ctx: &dyn DecompactionContext, + ) -> anyhow::Result { + try_map_array(compact.0, |v: T::Compact| T::decompact(v, ctx)) } } @@ -221,17 +133,17 @@ where count } - fn compact( - &self, - ctx: &mut CompactionContext, - ) -> anyhow::Result { + fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result { self.iter().map(|item| item.compact(ctx)).collect() } - fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result { + fn decompact( + compact: Self::Compact, + ctx: &dyn DecompactionContext, + ) -> anyhow::Result { compact .into_iter() - .map(|item| T::decompact(item, reg)) + .map(|item| T::decompact(item, ctx)) .collect() } } @@ -243,16 +155,13 @@ impl Compactable for PhantomData { CountPerTable::default() } - fn compact( - &self, - _ctx: &mut CompactionContext, - ) -> anyhow::Result { + fn compact(&self, _ctx: &mut dyn CompactionContext) -> anyhow::Result { Ok(()) } - fn decompact( + fn decompact( _compact: Self::Compact, - _reg: &R, + _ctx: &dyn DecompactionContext, ) -> anyhow::Result { Ok(Self) } @@ -266,7 +175,6 @@ mod tests { CompactionContext, CountPerTable, Key, - RegistryDb, }; use fuel_derive::Compact; use fuel_types::{ @@ -278,6 +186,8 @@ mod tests { Serialize, }; + use crate::DecompactionContext; + #[derive(Debug, Clone, PartialEq)] struct ManualExample { a: Address, @@ -299,21 +209,21 @@ mod tests { CountPerTable::Address(2) } - fn compact( + fn compact( &self, - ctx: &mut CompactionContext, + ctx: &mut dyn CompactionContext, ) -> anyhow::Result { - let a = ctx.to_key::(*self.a)?; - let b = ctx.to_key::(*self.b)?; + let a = ctx.to_key_Address(*self.a)?; + let b = ctx.to_key_Address(*self.b)?; Ok(ManualExampleCompact { a, b, c: self.c }) } - fn decompact( + fn decompact( compact: Self::Compact, - reg: &R, + ctx: &dyn DecompactionContext, ) -> anyhow::Result { - let a = Address::from(reg.read::(compact.a)?); - let b = Address::from(reg.read::(compact.b)?); + let a = Address::from(ctx.read_Address(compact.a)?); + let b = Address::from(ctx.read_Address(compact.b)?); Ok(Self { a, b, c: compact.c }) } } diff --git a/fuel-compression/src/registry/key.rs b/fuel-compression/src/key.rs similarity index 100% rename from fuel-compression/src/registry/key.rs rename to fuel-compression/src/key.rs diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index c2cc9d00d3..4b3ed43da5 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -6,23 +6,21 @@ #![deny(unused_crate_dependencies)] #![deny(clippy::cast_possible_truncation)] +mod block_section; mod compaction; -mod registry; +mod key; +mod table; -pub use compaction::{ - Compactable, - CompactionContext, -}; -pub use registry::{ +pub use compaction::Compactable; +pub use table::{ tables, ChangesPerTable, + CompactionContext, CountPerTable, - Key, - RegistryDb, + DecompactionContext, Table, }; -#[cfg(feature = "test-helpers")] -pub use registry::in_memory::InMemoryRegistry; +pub use key::Key; pub use fuel_derive::Compact; diff --git a/fuel-compression/src/registry/db.rs b/fuel-compression/src/registry/db.rs deleted file mode 100644 index 66c694da7e..0000000000 --- a/fuel-compression/src/registry/db.rs +++ /dev/null @@ -1,25 +0,0 @@ -use super::{ - Key, - Table, -}; - -/// Registry database needs these behaviors -pub trait RegistryDb { - /// Get next key for the given table. This is where the next write should start at. - /// The result of this function is just a suggestion, and the caller may choose to - /// ignore it, although it's rare that they would know better. - fn next_key(&self) -> anyhow::Result>; - - /// Read a value from the registry by key - fn read(&self, key: Key) -> anyhow::Result; - - /// Write a continuous sequence of values to the registry - fn batch_write( - &mut self, - start_key: Key, - values: Vec, - ) -> anyhow::Result<()>; - - /// Lookup a key by value - fn index_lookup(&self, value: &T::Type) -> anyhow::Result>>; -} diff --git a/fuel-compression/src/registry/in_memory.rs b/fuel-compression/src/registry/in_memory.rs deleted file mode 100644 index ba740f52a5..0000000000 --- a/fuel-compression/src/registry/in_memory.rs +++ /dev/null @@ -1,128 +0,0 @@ -use std::collections::HashMap; - -use super::{ - db::*, - key::RawKey, - Key, - Table, -}; - -/// Simple and inefficient in-memory registry for testing purposes. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct InMemoryRegistry { - next_keys: HashMap<&'static str, RawKey>, - storage: HashMap<&'static str, HashMap>>, - index: HashMap<&'static str, HashMap, RawKey>>, -} - -impl RegistryDb for InMemoryRegistry { - fn next_key(&self) -> anyhow::Result> { - Ok(Key::from_raw( - self.next_keys.get(T::NAME).copied().unwrap_or(RawKey::ZERO), - )) - } - - fn read(&self, key: Key) -> anyhow::Result { - if key == Key::DEFAULT_VALUE { - return Ok(T::Type::default()); - } - - Ok(self - .storage - .get(T::NAME) - .and_then(|table| table.get(&key.raw())) - .map(|bytes| postcard::from_bytes(bytes).expect("Invalid value in registry")) - .unwrap_or_default()) - } - - fn batch_write( - &mut self, - start_key: Key, - values: Vec, - ) -> anyhow::Result<()> { - let empty = values.is_empty(); - if !empty && start_key == Key::DEFAULT_VALUE { - panic!("Cannot write to the default value key"); - } - let table = self.storage.entry(T::NAME).or_default(); - let mut key = start_key.raw(); - for value in values.into_iter() { - let value = postcard::to_stdvec(&value).unwrap(); - let mut prefix = value.clone(); - prefix.truncate(32); - self.index.entry(T::NAME).or_default().insert(prefix, key); - table.insert(key, value); - key = key.next(); - } - if !empty { - self.next_keys.insert(T::NAME, key); - } - Ok(()) - } - - fn index_lookup(&self, value: &T::Type) -> anyhow::Result>> { - if *value == T::Type::default() { - return Ok(Some(Key::DEFAULT_VALUE)); - } - - let needle = postcard::to_stdvec(value).unwrap(); - let mut prefix = needle.clone(); - prefix.truncate(32); - - if let Some(cand) = self - .index - .get(T::NAME) - .and_then(|s| s.get(&prefix)) - .copied() - { - if let Some(cand_val) = self.storage.get(T::NAME).and_then(|s| s.get(&cand)) { - if *cand_val == needle { - return Ok(Some(Key::from_raw(cand))); - } - } - } - - Ok(None) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - tables, - Key, - }; - - #[test] - fn in_memory_registry_works() { - let mut reg = InMemoryRegistry::default(); - - // Empty - assert_eq!( - reg.read(Key::::try_from(100).unwrap()) - .unwrap(), - [0; 32] - ); - - // Write - reg.batch_write( - Key::::from_raw(RawKey::try_from(100u32).unwrap()), - vec![[1; 32], [2; 32]], - ) - .unwrap(); - - // Read - assert_eq!( - reg.read(Key::::try_from(100).unwrap()) - .unwrap(), - [1; 32] - ); - - // Index - assert_eq!( - reg.index_lookup(&[1; 32]).unwrap(), - Some(Key::::try_from(100).unwrap()) - ); - } -} diff --git a/fuel-compression/src/registry/mod.rs b/fuel-compression/src/table.rs similarity index 83% rename from fuel-compression/src/registry/mod.rs rename to fuel-compression/src/table.rs index 75b30aabbf..daa7694780 100644 --- a/fuel-compression/src/registry/mod.rs +++ b/fuel-compression/src/table.rs @@ -3,15 +3,9 @@ use serde::{ Serialize, }; -pub(crate) mod block_section; -pub mod db; -pub(crate) mod in_memory; -mod key; - -use self::block_section::WriteTo; -pub use self::{ - db::RegistryDb, - key::Key, +use crate::{ + block_section::WriteTo, + Key, }; mod _private { @@ -46,7 +40,7 @@ pub mod access { macro_rules! tables { // $index muse use increasing numbers starting from zero - ($($name:ident: $ty:ty),*$(,)?) => { + ($($name:ident: $ty:ty),*$(,)?) => { paste::paste! { /// Marker struct for each table type pub mod tables { $( @@ -63,6 +57,39 @@ macro_rules! tables { } type Type = $ty; } + + impl $name { + /// Calls the `to_key_*` method for this table on the context + pub fn to_key(value: $ty, ctx: &mut dyn super::CompactionContext) -> anyhow::Result> { + ctx.[](value) + } + + /// Calls the `read_*` method for this table on the context + pub fn read(key: super::Key<$name>, ctx: &dyn super::DecompactionContext) -> anyhow::Result<$ty> { + ctx.[](key) + } + } + )* + } + + /// Context for compaction, i.e. converting data to reference-based format + #[allow(non_snake_case)] // The field names match table type names eactly + #[allow(missing_docs)] // TODO + pub trait CompactionContext { + /// Store a value to the changeset and return a short reference key to it. + /// If the value already exists in the registry and will not be overwritten, + /// the existing key can be returned instead. + $( + fn [](&mut self, value: $ty) -> anyhow::Result>; + )* + } + + /// Context for compaction, i.e. converting data to reference-based format + #[allow(non_snake_case)] // The field names match table type names eactly + #[allow(missing_docs)] // TODO + pub trait DecompactionContext { + $( + fn [](&self, key: Key) -> anyhow::Result<::Type>; )* } @@ -145,12 +172,6 @@ macro_rules! tables { } )* - pub fn next_keys(reg: &mut R) -> anyhow::Result { - Ok(KeyPerTable { - $( $name: reg.next_key()?, )* - }) - } - /// Used to add together keys and counts to deterimine possible overwrite range pub fn add_keys(keys: KeyPerTable, counts: CountPerTable) -> KeyPerTable { KeyPerTable { @@ -186,14 +207,6 @@ macro_rules! tables { }),* } } - - /// Apply changes to the registry db - pub fn apply_to_registry(&self, reg: &mut R) -> anyhow::Result<()> { - $( - reg.batch_write(self.$name.start_key, self.$name.values.clone())?; - )* - Ok(()) - } } $( @@ -208,7 +221,7 @@ macro_rules! tables { } } )* - }; + }}; } tables!( diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compact.rs index 1df4f78091..addd92b8f0 100644 --- a/fuel-derive/src/compact.rs +++ b/fuel-derive/src/compact.rs @@ -117,7 +117,7 @@ impl FieldAttrs { } if let Ok(ident) = syn::parse2::(ml.tokens.clone()) { - if ident.to_string() == "skip" { + if ident == "skip" { result = Self::Skip; continue; } @@ -207,8 +207,9 @@ fn construct_compact( > }; quote! { - let #cname: #cty = ctx.to_key( - <#registry as Table>::Type::from(#binding.clone()) + let #cname: #cty = #registry::to_key( + <#registry as Table>::Type::from(#binding.clone()), + ctx, )?; } } @@ -264,13 +265,14 @@ fn construct_decompact( }, FieldAttrs::Normal => { quote! { - let #cname = <#ty as Compactable>::decompact(#binding, reg)?; + let #cname = <#ty as Compactable>::decompact(#binding, ctx)?; } } FieldAttrs::Registry(registry) => { quote! { - let raw: <#registry as Table>::Type = reg.read( - #binding + let raw: <#registry as Table>::Type = #registry::read( + #binding, + ctx, )?; let #cname = raw.into(); } @@ -478,7 +480,10 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { }); let impls = s.gen_impl(quote! { - use ::fuel_compression::{RegistryDb, tables, Table, Key, Compactable, CountPerTable, CompactionContext}; + use ::fuel_compression::{ + tables, Table, Key, Compactable, CountPerTable, + CompactionContext, DecompactionContext + }; gen impl Compactable for @Self #w_impl { type Compact = #compact_name #g; @@ -487,11 +492,11 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { match self { #count_per_variant } } - fn compact(&self, ctx: &mut CompactionContext) -> anyhow::Result { + fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result { Ok(match self { #construct_per_variant }) } - fn decompact(compact: Self::Compact, reg: &R) -> anyhow::Result { + fn decompact(compact: Self::Compact, ctx: &dyn DecompactionContext) -> anyhow::Result { Ok(match compact { #decompact_per_variant }) } } diff --git a/fuel-tx/src/transaction/types/input.rs b/fuel-tx/src/transaction/types/input.rs index 68dbc6dcfc..e4fb16cd46 100644 --- a/fuel-tx/src/transaction/types/input.rs +++ b/fuel-tx/src/transaction/types/input.rs @@ -112,16 +112,16 @@ where Default::default() } - fn compact( + fn compact( &self, - _: &mut fuel_compression::CompactionContext, + _: &mut dyn fuel_compression::CompactionContext, ) -> anyhow::Result { Ok(()) } - fn decompact( + fn decompact( _: Self::Compact, - _: &R, + _: &dyn fuel_compression::DecompactionContext, ) -> anyhow::Result { Ok(Self(Default::default())) } diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index 694a39c06c..266d80ae29 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -90,7 +90,7 @@ profile-gas = ["profile-any"] profile-coverage = ["profile-any"] profile-any = ["dyn-clone"] # All profiling features should depend on this random = ["fuel-crypto/random", "fuel-types/random", "fuel-tx/random", "rand"] -da-compression = ["dep:anyhow", "fuel-compression", "fuel-tx/da-compression"] +da-compression = ["fuel-compression", "fuel-tx/da-compression"] serde = [ "dep:serde", "dep:serde_with", From 546dda067e74198bd0f96807d5caa8601f7b1dae Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 22 Jul 2024 12:01:51 +0300 Subject: [PATCH 17/77] Re-add dummy registry for testing --- fuel-compression/Cargo.toml | 6 +- fuel-compression/src/block_section.rs | 2 +- fuel-compression/src/compaction.rs | 21 ++- fuel-compression/src/dummy_registry.rs | 213 +++++++++++++++++++++++++ fuel-compression/src/key.rs | 5 + fuel-compression/src/lib.rs | 8 +- fuel-compression/src/table.rs | 100 +----------- 7 files changed, 247 insertions(+), 108 deletions(-) create mode 100644 fuel-compression/src/dummy_registry.rs diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index 89b85636bb..c6fd089a4d 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -16,12 +16,14 @@ fuel-derive = { workspace = true } paste = "1.0" serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" +postcard = { version = "1.0", features = ["use-std"], optional = true} [dev-dependencies] +bincode = { workspace = true } fuel-asm = { workspace = true } -fuel-compression = { path = ".", features = ["test-helpers"] } # Self-dependency needed by test for macros fuel-tx = { workspace = true } +fuel-compression = { workspace = true, features = ["test-helpers"]} fuel-types = { workspace = true, features = ["serde"] } [features] -test-helpers = [] +test-helpers = ["dep:postcard"] diff --git a/fuel-compression/src/block_section.rs b/fuel-compression/src/block_section.rs index 6bf912457a..22d7ce07aa 100644 --- a/fuel-compression/src/block_section.rs +++ b/fuel-compression/src/block_section.rs @@ -17,7 +17,7 @@ use super::{ pub struct WriteTo { /// The values are inserted starting from this key pub start_key: Key, - /// Values. inserted using incrementing ids starting from `start_key` + /// Values, inserted using incrementing ids starting from `start_key` pub values: Vec, } diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 781c77d9fd..595c1775dd 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -12,7 +12,6 @@ use crate::{ CompactionContext, CountPerTable, DecompactionContext, - Key, }; /// Convert data to reference-based format @@ -170,10 +169,12 @@ impl Compactable for PhantomData { #[cfg(test)] mod tests { use fuel_compression::{ + dummy_registry::DummyRegistry, tables, Compactable, CompactionContext, CountPerTable, + DecompactionContext, Key, }; use fuel_derive::Compact; @@ -186,8 +187,6 @@ mod tests { Serialize, }; - use crate::DecompactionContext; - #[derive(Debug, Clone, PartialEq)] struct ManualExample { a: Address, @@ -257,27 +256,27 @@ mod tests { } #[test] - fn test_compaction_roundtrip() { + fn test_compaction_roundtrip_manual() { let target = ManualExample { a: Address::from([1u8; 32]), b: Address::from([2u8; 32]), c: 3, }; - let mut registry = fuel_compression::InMemoryRegistry::default(); - let (compacted, _) = - CompactionContext::run(&mut registry, target.clone()).unwrap(); + let mut registry = DummyRegistry::default(); + let (compacted, _) = registry.compact(target.clone()).unwrap(); let decompacted = ManualExample::decompact(compacted, ®istry).unwrap(); assert_eq!(target, decompacted); + } + #[test] + fn test_compaction_roundtrip_derive() { let target = AutomaticExample { a: AssetId::from([1u8; 32]), b: AssetId::from([2u8; 32]), c: 3, }; - let mut registry = fuel_compression::InMemoryRegistry::default(); - let (compacted, _) = - fuel_compression::CompactionContext::run(&mut registry, target.clone()) - .unwrap(); + let mut registry = DummyRegistry::default(); + let (compacted, _) = registry.compact(target.clone()).unwrap(); let decompacted = AutomaticExample::decompact(compacted, ®istry).unwrap(); assert_eq!(target, decompacted); } diff --git a/fuel-compression/src/dummy_registry.rs b/fuel-compression/src/dummy_registry.rs new file mode 100644 index 0000000000..822c35cb81 --- /dev/null +++ b/fuel-compression/src/dummy_registry.rs @@ -0,0 +1,213 @@ +//! Temporal registry implementation used for tests + +use std::collections::HashMap; + +use crate::{ + block_section::WriteTo, + key::RawKey, + table::{ + access::*, + add_keys, + KeyPerTable, + }, + tables, + ChangesPerTable, + Compactable, + CompactionContext, + DecompactionContext, + Key, + Table, +}; + +type TableName = &'static str; + +/// Temporal registry implementation used for tests. +#[derive(Default)] +pub struct DummyRegistry { + next_keys: KeyPerTable, + values: HashMap<(TableName, RawKey), Vec>, +} + +impl DummyRegistry { + /// Run the compaction for the given target, returning the compacted data. + /// Applies the changes to the registry, but also returns them for block inclusion. + pub fn compact( + &mut self, + target: C, + ) -> anyhow::Result<(C::Compact, ChangesPerTable)> { + let key_limits = target.count(); + let safe_keys_start = add_keys(self.next_keys, key_limits); + + let mut ctx = DummyCompactionCtx { + start_keys: self.next_keys, + next_keys: self.next_keys, + safe_keys_start, + changes: ChangesPerTable::from_start_keys(self.next_keys), + reg: self, + }; + + let compacted = target.compact(&mut ctx)?; + let changes = ctx.finalize(); + self.apply_changes(&changes); + Ok((compacted, changes)) + } + + fn apply_changes(&mut self, changes: &ChangesPerTable) { + macro_rules! for_tables { + ($($name:ident),*$(,)?) => { paste::paste! {{ + #[allow(non_snake_case)] + let ChangesPerTable { + $($name: [<$name _changes>],)* + } = changes; + + $( + let mut key = [< $name _changes >].start_key.raw(); + for value in [< $name _changes >].values.iter() { + self.values.insert((tables::$name::NAME, key), postcard::to_stdvec(&value).unwrap()); + key = key.next(); + } + )* + } }}; + } + + for_tables!(AssetId, Address, ContractId, ScriptCode, Witness) + } + + fn resolve_key(&self, key: Key) -> anyhow::Result { + self.values + .get(&(::NAME, key.raw())) + .ok_or_else(|| anyhow::anyhow!("Key not found: {:?}", key)) + .and_then(|bytes| postcard::from_bytes(bytes).map_err(|e| e.into())) + } +} + +/// Compaction session for +pub struct DummyCompactionCtx<'a> { + /// The registry + reg: &'a DummyRegistry, + /// These are the keys where writing started + start_keys: KeyPerTable, + /// The next keys to use for each table + next_keys: KeyPerTable, + /// Keys in range next_keys..safe_keys_start + /// could be overwritten by the compaction, + /// and cannot be used for new values. + safe_keys_start: KeyPerTable, + changes: ChangesPerTable, +} + +impl<'a> DummyCompactionCtx<'a> { + /// Convert a value to a key + /// If necessary, store the value in the changeset and allocate a new key. + fn value_to_key(&mut self, value: T::Type) -> anyhow::Result> + where + KeyPerTable: AccessCopy> + AccessMut>, + ChangesPerTable: AccessRef> + AccessMut>, + { + // Check if the value is within the current changeset + if let Some(key) = + >>::get(&self.changes) + .lookup_value(&value) + { + return Ok(key); + } + + // Check if the registry contains this value already. + // This is a slow linear search, but since this is test-only code, it's ok. + let encoded = postcard::to_stdvec(&value).unwrap(); + for ((table_name, raw_key), bytes) in self.reg.values.iter() { + if *table_name == ::NAME && *bytes == encoded { + let key = Key::::from_raw(*raw_key); + // Check if the value is in the possibly-overwritable range + let start: Key = self.start_keys.value(); + let end: Key = self.safe_keys_start.value(); + if !key.is_between(start, end) { + return Ok(key); + } + } + } + + // Allocate a new key for this + let key = >>::get_mut(&mut self.next_keys) + .take_next(); + >>::get_mut(&mut self.changes) + .values + .push(value); + Ok(key) + } +} + +impl<'a> CompactionContext for DummyCompactionCtx<'a> { + fn finalize(self) -> ChangesPerTable { + self.changes + } + + fn to_key_AssetId( + &mut self, + value: [u8; 32], + ) -> anyhow::Result> { + self.value_to_key(value) + } + + fn to_key_Address( + &mut self, + value: [u8; 32], + ) -> anyhow::Result> { + self.value_to_key(value) + } + + fn to_key_ContractId( + &mut self, + value: [u8; 32], + ) -> anyhow::Result> { + self.value_to_key(value) + } + + fn to_key_ScriptCode( + &mut self, + value: Vec, + ) -> anyhow::Result> { + self.value_to_key(value) + } + + fn to_key_Witness(&mut self, value: Vec) -> anyhow::Result> { + self.value_to_key(value) + } +} + +impl DecompactionContext for DummyRegistry { + fn read_AssetId( + &self, + key: Key, + ) -> anyhow::Result<::Type> { + self.resolve_key(key) + } + + fn read_Address( + &self, + key: Key, + ) -> anyhow::Result<::Type> { + self.resolve_key(key) + } + + fn read_ContractId( + &self, + key: Key, + ) -> anyhow::Result<::Type> { + self.resolve_key(key) + } + + fn read_ScriptCode( + &self, + key: Key, + ) -> anyhow::Result<::Type> { + self.resolve_key(key) + } + + fn read_Witness( + &self, + key: Key, + ) -> anyhow::Result<::Type> { + self.resolve_key(key) + } +} diff --git a/fuel-compression/src/key.rs b/fuel-compression/src/key.rs index 88ee5072af..08c391512c 100644 --- a/fuel-compression/src/key.rs +++ b/fuel-compression/src/key.rs @@ -14,11 +14,16 @@ use super::Table; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct RawKey([u8; Self::SIZE]); impl RawKey { + /// Key mapping to default value for the table type. pub const DEFAULT_VALUE: Self = Self([u8::MAX; Self::SIZE]); + /// Maximum writable key. pub const MAX_WRITABLE: Self = Self([u8::MAX, u8::MAX, u8::MAX - 1]); + /// Size of the key, in bytes. pub const SIZE: usize = 3; + /// Zero key. pub const ZERO: Self = Self([0; Self::SIZE]); + /// Convert to u32, big-endian. pub fn as_u32(self) -> u32 { u32::from_be_bytes([0, self.0[0], self.0[1], self.0[2]]) } diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index 4b3ed43da5..6f49dbbfc0 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -11,6 +11,9 @@ mod compaction; mod key; mod table; +#[cfg(any(test, feature = "test-helpers"))] +pub mod dummy_registry; + pub use compaction::Compactable; pub use table::{ tables, @@ -21,6 +24,9 @@ pub use table::{ Table, }; -pub use key::Key; +pub use key::{ + Key, + RawKey, +}; pub use fuel_derive::Compact; diff --git a/fuel-compression/src/table.rs b/fuel-compression/src/table.rs index daa7694780..d4bf138c2a 100644 --- a/fuel-compression/src/table.rs +++ b/fuel-compression/src/table.rs @@ -72,10 +72,16 @@ macro_rules! tables { )* } - /// Context for compaction, i.e. converting data to reference-based format + /// Context for compaction, i.e. converting data to reference-based format. + /// The context is used to aggreage changes to the registry. + /// A new context should be created for each compaction "session", + /// typically a blockchain block. #[allow(non_snake_case)] // The field names match table type names eactly #[allow(missing_docs)] // TODO pub trait CompactionContext { + /// Ends session, returning the changeset. + fn finalize(self) -> ChangesPerTable; + /// Store a value to the changeset and return a short reference key to it. /// If the value already exists in the registry and will not be overwritten, /// the existing key can be returned instead. @@ -231,95 +237,3 @@ tables!( ScriptCode: Vec, Witness: Vec, ); - -#[cfg(test)] -mod tests { - use fuel_types::AssetId; - use tests::key::RawKey; - - use super::*; - - #[test] - fn test_in_memory_db() { - let mut reg = in_memory::InMemoryRegistry::default(); - - // Empty - assert_eq!( - reg.read(Key::::try_from(100).unwrap()) - .unwrap(), - [0; 32] - ); - assert_eq!( - reg.index_lookup(&*AssetId::from([1; 32])).unwrap(), - None::> - ); - - // Write - reg.batch_write( - Key::::from_raw(RawKey::try_from(100u32).unwrap()), - vec![[1; 32], [2; 32]], - ) - .unwrap(); - assert_eq!( - reg.read(Key::::try_from(100).unwrap()) - .unwrap(), - [1; 32] - ); - assert_eq!( - reg.read(Key::::try_from(101).unwrap()) - .unwrap(), - [2; 32] - ); - assert_eq!( - reg.read(Key::::try_from(102).unwrap()) - .unwrap(), - [0; 32] - ); - - // Overwrite - reg.batch_write( - Key::::from_raw(RawKey::try_from(99u32).unwrap()), - vec![[10; 32], [11; 32]], - ) - .unwrap(); - assert_eq!( - reg.read(Key::::try_from(99).unwrap()) - .unwrap(), - [10; 32] - ); - assert_eq!( - reg.read(Key::::try_from(100).unwrap()) - .unwrap(), - [11; 32] - ); - - // Wrapping - reg.batch_write( - Key::::from_raw(RawKey::MAX_WRITABLE), - vec![[3; 32], [4; 32]], - ) - .unwrap(); - - assert_eq!( - reg.read(Key::::from_raw(RawKey::MAX_WRITABLE)) - .unwrap(), - [3; 32] - ); - - assert_eq!( - reg.read(Key::::from_raw(RawKey::ZERO)) - .unwrap(), - [4; 32] - ); - - assert_eq!( - reg.index_lookup(&*AssetId::from([3; 32])).unwrap(), - Some(Key::::from_raw(RawKey::MAX_WRITABLE)) - ); - - assert_eq!( - reg.index_lookup(&*AssetId::from([4; 32])).unwrap(), - Some(Key::::from_raw(RawKey::ZERO)) - ); - } -} From bf2b97611aea7a12766a7c06b564e64c926b8472 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 22 Jul 2024 12:06:07 +0300 Subject: [PATCH 18/77] Add some missing docs --- fuel-compression/src/table.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fuel-compression/src/table.rs b/fuel-compression/src/table.rs index d4bf138c2a..ea9eb60247 100644 --- a/fuel-compression/src/table.rs +++ b/fuel-compression/src/table.rs @@ -77,24 +77,23 @@ macro_rules! tables { /// A new context should be created for each compaction "session", /// typically a blockchain block. #[allow(non_snake_case)] // The field names match table type names eactly - #[allow(missing_docs)] // TODO pub trait CompactionContext { /// Ends session, returning the changeset. fn finalize(self) -> ChangesPerTable; - /// Store a value to the changeset and return a short reference key to it. - /// If the value already exists in the registry and will not be overwritten, - /// the existing key can be returned instead. $( + /// Store a value to the changeset and return a short reference key to it. + /// If the value already exists in the registry and will not be overwritten, + /// the existing key can be returned instead. fn [](&mut self, value: $ty) -> anyhow::Result>; )* } /// Context for compaction, i.e. converting data to reference-based format #[allow(non_snake_case)] // The field names match table type names eactly - #[allow(missing_docs)] // TODO pub trait DecompactionContext { $( + /// Read a value from the registry based on the key. fn [](&self, key: Key) -> anyhow::Result<::Type>; )* } From 880d13c67e7a30c76a13ff2db97b787bdd62eafb Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 22 Jul 2024 12:34:31 +0300 Subject: [PATCH 19/77] Move block section data for fuel-core --- fuel-compression/Cargo.toml | 4 +- fuel-compression/src/block_section.rs | 204 ------------------------- fuel-compression/src/dummy_registry.rs | 72 ++++----- fuel-compression/src/lib.rs | 2 - fuel-compression/src/table.rs | 47 +----- 5 files changed, 34 insertions(+), 295 deletions(-) delete mode 100644 fuel-compression/src/block_section.rs diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index c6fd089a4d..485f3b3a7b 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -19,9 +19,7 @@ serde-big-array = "0.5" postcard = { version = "1.0", features = ["use-std"], optional = true} [dev-dependencies] -bincode = { workspace = true } -fuel-asm = { workspace = true } -fuel-tx = { workspace = true } +# bincode = { workspace = true } fuel-compression = { workspace = true, features = ["test-helpers"]} fuel-types = { workspace = true, features = ["serde"] } diff --git a/fuel-compression/src/block_section.rs b/fuel-compression/src/block_section.rs deleted file mode 100644 index 22d7ce07aa..0000000000 --- a/fuel-compression/src/block_section.rs +++ /dev/null @@ -1,204 +0,0 @@ -use core::fmt; - -use serde::{ - ser::SerializeTuple, - Deserialize, - Serialize, -}; - -use super::{ - key::Key, - ChangesPerTable, - Table, -}; - -/// New registrations written to a specific table. -#[derive(Clone, PartialEq, Eq)] -pub struct WriteTo { - /// The values are inserted starting from this key - pub start_key: Key, - /// Values, inserted using incrementing ids starting from `start_key` - pub values: Vec, -} - -impl fmt::Debug for WriteTo -where - T::Type: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.values.is_empty() { - return f.write_str("WriteTo::EMPTY"); - } - - f.debug_struct("WriteTo") - .field("start_key", &self.start_key) - .field("values", &self.values) - .finish() - } -} - -impl WriteTo -where - T::Type: PartialEq, -{ - /// Reverse lookup. - /// TODO: possibly add a lookup table for this, if deemed necessary - pub fn lookup_value(&self, needle: &T::Type) -> Option> { - if *needle == T::Type::default() { - return Some(Key::DEFAULT_VALUE); - } - - let mut key = self.start_key; - for v in &self.values { - if v == needle { - return Some(key); - } - key = key.next(); - } - None - } -} - -/// Custom serialization is used to omit the start_key when the sequence is empty -impl Serialize for WriteTo -where - T: Table + Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - let mut tup = serializer.serialize_tuple(2)?; - tup.serialize_element(&self.values)?; - if self.values.is_empty() { - tup.serialize_element(&())?; - } else { - tup.serialize_element(&self.start_key)?; - } - tup.end() - } -} - -impl<'de, T: Table> Deserialize<'de> for WriteTo -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_tuple( - 2, - Self { - start_key: Key::ZERO, - values: Vec::new(), - }, - ) - } -} - -impl<'de, T: Table + Deserialize<'de>> serde::de::Visitor<'de> for WriteTo { - type Value = WriteTo; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str(concat!("WriteTo<", stringify!(T), "> instance")) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let values: Vec = seq.next_element()?.ok_or( - serde::de::Error::invalid_length(0, &"WriteTo<_> with 2 elements"), - )?; - - if values.is_empty() { - seq.next_element()?.ok_or(serde::de::Error::invalid_length( - 1, - &"WriteTo<_> with 2 elements", - ))?; - Ok(WriteTo { - start_key: Key::ZERO, - values, - }) - } else { - let start_key: Key = seq.next_element()?.ok_or( - serde::de::Error::invalid_length(1, &"WriteTo<_> with 2 elements"), - )?; - Ok(WriteTo { start_key, values }) - } - } -} - -/// Registeration section of the compressed block -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Registrations { - /// Merkle root of the registeration table merkle roots - pub tables_root: [u8; 32], - /// Changes per table - pub changes: ChangesPerTable, -} - -#[cfg(test)] -mod tests { - use super::*; - use bincode::Options; - use fuel_asm::op; - use fuel_tx::{ - AssetId, - ContractId, - }; - use fuel_types::Address; - - #[test] - fn test_tables() { - let original = Registrations { - tables_root: Default::default(), - changes: ChangesPerTable { - AssetId: WriteTo { - start_key: Key::try_from(100).unwrap(), - values: vec![*AssetId::from([0xa0; 32]), *AssetId::from([0xa1; 32])], - }, - Address: WriteTo { - start_key: Key::ZERO, - values: vec![*Address::from([0xb0; 32])], - }, - ContractId: WriteTo { - start_key: Key::ZERO, - values: vec![*ContractId::from([0xc0; 32])], - }, - ScriptCode: WriteTo { - start_key: Key::ZERO, - values: vec![ - vec![op::addi(0x20, 0x20, 1), op::ret(0)] - .into_iter() - .collect(), - vec![op::muli(0x20, 0x20, 5), op::ret(1)] - .into_iter() - .collect(), - ], - }, - Witness: WriteTo { - start_key: Key::ZERO, - values: vec![], - }, - }, - }; - - let pc_compressed = postcard::to_stdvec(&original).unwrap(); - let pc_decompressed: Registrations = - postcard::from_bytes(&pc_compressed).unwrap(); - assert_eq!(original, pc_decompressed); - - let bc_opt = bincode::DefaultOptions::new().with_varint_encoding(); - - let bc_compressed = bc_opt.serialize(&original).unwrap(); - let bc_decompressed: Registrations = bc_opt.deserialize(&bc_compressed).unwrap(); - assert_eq!(original, bc_decompressed); - - println!("data: {original:?}"); - println!("postcard compressed size {}", pc_compressed.len()); - println!("bincode compressed size {}", bc_compressed.len()); - println!("postcard compressed: {:x?}", pc_compressed); - println!("bincode compressed: {:x?}", bc_compressed); - - // panic!("ok, just showing the results"); - } -} diff --git a/fuel-compression/src/dummy_registry.rs b/fuel-compression/src/dummy_registry.rs index 822c35cb81..84ed2d3cdb 100644 --- a/fuel-compression/src/dummy_registry.rs +++ b/fuel-compression/src/dummy_registry.rs @@ -3,7 +3,6 @@ use std::collections::HashMap; use crate::{ - block_section::WriteTo, key::RawKey, table::{ access::*, @@ -11,7 +10,6 @@ use crate::{ KeyPerTable, }, tables, - ChangesPerTable, Compactable, CompactionContext, DecompactionContext, @@ -34,7 +32,7 @@ impl DummyRegistry { pub fn compact( &mut self, target: C, - ) -> anyhow::Result<(C::Compact, ChangesPerTable)> { + ) -> anyhow::Result<(C::Compact, Changes)> { let key_limits = target.count(); let safe_keys_start = add_keys(self.next_keys, key_limits); @@ -42,35 +40,16 @@ impl DummyRegistry { start_keys: self.next_keys, next_keys: self.next_keys, safe_keys_start, - changes: ChangesPerTable::from_start_keys(self.next_keys), + changes: Changes::default(), reg: self, }; let compacted = target.compact(&mut ctx)?; - let changes = ctx.finalize(); - self.apply_changes(&changes); - Ok((compacted, changes)) - } - - fn apply_changes(&mut self, changes: &ChangesPerTable) { - macro_rules! for_tables { - ($($name:ident),*$(,)?) => { paste::paste! {{ - #[allow(non_snake_case)] - let ChangesPerTable { - $($name: [<$name _changes>],)* - } = changes; - - $( - let mut key = [< $name _changes >].start_key.raw(); - for value in [< $name _changes >].values.iter() { - self.values.insert((tables::$name::NAME, key), postcard::to_stdvec(&value).unwrap()); - key = key.next(); - } - )* - } }}; + let changes = ctx.changes; + for change in changes.changes.iter() { + self.values.insert((change.0, change.1), change.2.clone()); } - - for_tables!(AssetId, Address, ContractId, ScriptCode, Witness) + Ok((compacted, changes)) } fn resolve_key(&self, key: Key) -> anyhow::Result { @@ -81,6 +60,25 @@ impl DummyRegistry { } } +/// Changeset for the registry +#[derive(Default)] +pub struct Changes { + changes: Vec<(TableName, RawKey, Vec)>, +} +impl Changes { + fn lookup_value(&self, value: &::Type) -> Option> { + // Slow linear search. This is test-only code, so it's ok. + for change in self.changes.iter() { + if change.0 == ::NAME + && change.2 == postcard::to_stdvec(value).unwrap() + { + return Some(Key::::from_raw(change.1)); + } + } + None + } +} + /// Compaction session for pub struct DummyCompactionCtx<'a> { /// The registry @@ -93,7 +91,7 @@ pub struct DummyCompactionCtx<'a> { /// could be overwritten by the compaction, /// and cannot be used for new values. safe_keys_start: KeyPerTable, - changes: ChangesPerTable, + changes: Changes, } impl<'a> DummyCompactionCtx<'a> { @@ -102,13 +100,9 @@ impl<'a> DummyCompactionCtx<'a> { fn value_to_key(&mut self, value: T::Type) -> anyhow::Result> where KeyPerTable: AccessCopy> + AccessMut>, - ChangesPerTable: AccessRef> + AccessMut>, { // Check if the value is within the current changeset - if let Some(key) = - >>::get(&self.changes) - .lookup_value(&value) - { + if let Some(key) = self.changes.lookup_value(&value) { return Ok(key); } @@ -130,18 +124,16 @@ impl<'a> DummyCompactionCtx<'a> { // Allocate a new key for this let key = >>::get_mut(&mut self.next_keys) .take_next(); - >>::get_mut(&mut self.changes) - .values - .push(value); + self.changes.changes.push(( + ::NAME, + key.raw(), + postcard::to_stdvec(&value).unwrap(), + )); Ok(key) } } impl<'a> CompactionContext for DummyCompactionCtx<'a> { - fn finalize(self) -> ChangesPerTable { - self.changes - } - fn to_key_AssetId( &mut self, value: [u8; 32], diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index 6f49dbbfc0..083ecd28e9 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -6,7 +6,6 @@ #![deny(unused_crate_dependencies)] #![deny(clippy::cast_possible_truncation)] -mod block_section; mod compaction; mod key; mod table; @@ -17,7 +16,6 @@ pub mod dummy_registry; pub use compaction::Compactable; pub use table::{ tables, - ChangesPerTable, CompactionContext, CountPerTable, DecompactionContext, diff --git a/fuel-compression/src/table.rs b/fuel-compression/src/table.rs index ea9eb60247..71982222d4 100644 --- a/fuel-compression/src/table.rs +++ b/fuel-compression/src/table.rs @@ -3,10 +3,7 @@ use serde::{ Serialize, }; -use crate::{ - block_section::WriteTo, - Key, -}; +use crate::Key; mod _private { pub trait Seal {} @@ -78,9 +75,6 @@ macro_rules! tables { /// typically a blockchain block. #[allow(non_snake_case)] // The field names match table type names eactly pub trait CompactionContext { - /// Ends session, returning the changeset. - fn finalize(self) -> ChangesPerTable; - $( /// Store a value to the changeset and return a short reference key to it. /// If the value already exists in the registry and will not be overwritten, @@ -187,45 +181,6 @@ macro_rules! tables { )* } } - - /// Registeration changes per table - #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] - #[allow(non_snake_case)] // The field names match table type names eactly - #[allow(missing_docs)] // Makes no sense to document the fields - #[non_exhaustive] - pub struct ChangesPerTable { - $(pub $name: WriteTo),* - } - - impl ChangesPerTable { - /// Is the changeset empty for all tables? - pub fn is_empty(&self) -> bool { - true $(&& self.$name.values.is_empty())* - } - - /// Create a new changeset with the given start keys - pub fn from_start_keys(start_keys: KeyPerTable) -> Self { - Self { - $($name: WriteTo { - start_key: start_keys.$name, - values: Vec::new(), - }),* - } - } - } - - $( - impl access::AccessRef> for ChangesPerTable { - fn get(&self) -> &WriteTo { - &self.$name - } - } - impl access::AccessMut> for ChangesPerTable { - fn get_mut(&mut self) -> &mut WriteTo { - &mut self.$name - } - } - )* }}; } From b23bff55cde3a9cb0ea998b63d889b19728bad1c Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 22 Jul 2024 13:47:19 +0300 Subject: [PATCH 20/77] Minor change for fuel-core integration --- fuel-compression/src/dummy_registry.rs | 6 ++-- fuel-compression/src/lib.rs | 3 ++ fuel-compression/src/table.rs | 47 ++++++++++++++++++++------ 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/fuel-compression/src/dummy_registry.rs b/fuel-compression/src/dummy_registry.rs index 84ed2d3cdb..894cf82638 100644 --- a/fuel-compression/src/dummy_registry.rs +++ b/fuel-compression/src/dummy_registry.rs @@ -6,7 +6,6 @@ use crate::{ key::RawKey, table::{ access::*, - add_keys, KeyPerTable, }, tables, @@ -15,10 +14,9 @@ use crate::{ DecompactionContext, Key, Table, + TableName, }; -type TableName = &'static str; - /// Temporal registry implementation used for tests. #[derive(Default)] pub struct DummyRegistry { @@ -34,7 +32,7 @@ impl DummyRegistry { target: C, ) -> anyhow::Result<(C::Compact, Changes)> { let key_limits = target.count(); - let safe_keys_start = add_keys(self.next_keys, key_limits); + let safe_keys_start = self.next_keys.offset_by(key_limits); let mut ctx = DummyCompactionCtx { start_keys: self.next_keys, diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index 083ecd28e9..ac7969b937 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -15,11 +15,14 @@ pub mod dummy_registry; pub use compaction::Compactable; pub use table::{ + access, tables, CompactionContext, CountPerTable, DecompactionContext, + KeyPerTable, Table, + TableName, }; pub use key::{ diff --git a/fuel-compression/src/table.rs b/fuel-compression/src/table.rs index 71982222d4..391a080ddd 100644 --- a/fuel-compression/src/table.rs +++ b/fuel-compression/src/table.rs @@ -3,16 +3,22 @@ use serde::{ Serialize, }; -use crate::Key; +use crate::{ + Key, + RawKey, +}; mod _private { pub trait Seal {} } +/// Static name of a table +pub type TableName = &'static str; + /// Table in the registry pub trait Table: _private::Seal { /// Unique name of the table - const NAME: &'static str; + const NAME: TableName; /// A `CountPerTable` for this table fn count(n: usize) -> CountPerTable; @@ -21,22 +27,28 @@ pub trait Table: _private::Seal { type Type: PartialEq + Default + Serialize + for<'de> Deserialize<'de>; } +/// Traits for accessing `*PerTable` using the table type pub mod access { + /// Copy value for the give table pub trait AccessCopy { + /// Copy value for the give table fn value(&self) -> V; } + /// Get reference to the value for the given table pub trait AccessRef { + /// Get reference to the value for the given table fn get(&self) -> &V; } + /// Get mutable reference to the value for the given table pub trait AccessMut { + /// Get mutable reference to the value for the given table fn get_mut(&mut self) -> &mut V; } } macro_rules! tables { - // $index muse use increasing numbers starting from zero ($($name:ident: $ty:ty),*$(,)?) => { paste::paste! { /// Marker struct for each table type pub mod tables { @@ -153,6 +165,16 @@ macro_rules! tables { } } + impl KeyPerTable { + /// Generate keys for each table using a function that's called for each table. + /// Since type erasure is required, the function takes `TableName` adn returns `RawKey`. + pub fn from_fn RawKey>(f: F) -> Self { + Self { + $($name: Key::from_raw(f(::NAME)),)* + } + } + } + $( impl access::AccessCopy> for KeyPerTable { fn value(&self) -> Key { @@ -171,14 +193,17 @@ macro_rules! tables { } )* - /// Used to add together keys and counts to deterimine possible overwrite range - pub fn add_keys(keys: KeyPerTable, counts: CountPerTable) -> KeyPerTable { - KeyPerTable { - $( - $name: keys.$name.add_u32(counts.$name.try_into() - .expect("Count too large. Shoudn't happen as we control inputs here.") - ), - )* + impl KeyPerTable { + /// Used to add together keys and counts to deterimine possible overwrite range. + /// Panics if the keys count cannot fit into `u32`. + pub fn offset_by(&self, counts: CountPerTable) -> KeyPerTable { + KeyPerTable { + $( + $name: self.$name.add_u32(counts.$name.try_into() + .expect("Count too large. Shouldn't happen as we control inputs here.") + ), + )* + } } } }}; From 293f0f0bef158c32bc9bd3fa49911f07d80be79e Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 22 Jul 2024 19:57:38 +0300 Subject: [PATCH 21/77] cargo sort --- fuel-compression/Cargo.toml | 4 ++-- fuel-vm/Cargo.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index 485f3b3a7b..dc12d92181 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -14,13 +14,13 @@ description = "Compression and decompression of Fuel blocks for DA storage." anyhow = { workspace = true } fuel-derive = { workspace = true } paste = "1.0" +postcard = { version = "1.0", features = ["use-std"], optional = true } serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" -postcard = { version = "1.0", features = ["use-std"], optional = true} [dev-dependencies] # bincode = { workspace = true } -fuel-compression = { workspace = true, features = ["test-helpers"]} +fuel-compression = { workspace = true, features = ["test-helpers"] } fuel-types = { workspace = true, features = ["serde"] } [features] diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index 266d80ae29..f04f840e21 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -108,4 +108,5 @@ test-helpers = [ "dep:anyhow", "tai64", "fuel-crypto/test-helpers", + "fuel-compression?/test-helpers", ] From a1b5174d1721bea90549279dc8b9ff673af8b1f3 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 23 Jul 2024 08:44:39 +0300 Subject: [PATCH 22/77] Fix incorrect feature cfg on default_test_tx --- fuel-tx/src/transaction.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuel-tx/src/transaction.rs b/fuel-tx/src/transaction.rs index a1e9d78474..3bc8f6788e 100644 --- a/fuel-tx/src/transaction.rs +++ b/fuel-tx/src/transaction.rs @@ -112,7 +112,7 @@ impl Default for Transaction { impl Transaction { /// Return default valid transaction useful for tests. - #[cfg(all(feature = "rand", feature = "std", feature = "test-helpers"))] + #[cfg(all(feature = "random", feature = "std", feature = "test-helpers"))] pub fn default_test_tx() -> Self { use crate::Finalizable; From f7d6cedd1589d9e9fb4fb47b16e18167a0b6ac75 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 27 Aug 2024 05:09:21 +0300 Subject: [PATCH 23/77] Implement compacting for blobs as well --- fuel-tx/src/transaction/types/blob.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/fuel-tx/src/transaction/types/blob.rs b/fuel-tx/src/transaction/types/blob.rs index 9eb9210461..888a11854d 100644 --- a/fuel-tx/src/transaction/types/blob.rs +++ b/fuel-tx/src/transaction/types/blob.rs @@ -47,6 +47,7 @@ pub struct BlobMetadata; /// The body of the [`Blob`] transaction. #[derive(Clone, Default, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Blob)] #[derivative(Eq, PartialEq, Hash, Debug)] From 21b4ab6b3f85c6f2ccd4daba454a9d3b3bcceb2c Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 27 Aug 2024 05:25:02 +0300 Subject: [PATCH 24/77] Fix no_std deps --- Cargo.toml | 1 - fuel-asm/Cargo.toml | 2 +- fuel-compression/Cargo.toml | 3 +-- fuel-tx/Cargo.toml | 2 +- fuel-types/Cargo.toml | 6 +++--- fuel-vm/Cargo.toml | 2 +- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0ce6187d30..aa6a4fb690 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ repository = "https://github.com/FuelLabs/fuel-vm" version = "0.56.0" [workspace.dependencies] -anyhow = "1.0" fuel-asm = { version = "0.56.0", path = "fuel-asm", default-features = false } fuel-crypto = { version = "0.56.0", path = "fuel-crypto", default-features = false } fuel-compression = { version = "0.56.0", path = "fuel-compression", default-features = false } diff --git a/fuel-asm/Cargo.toml b/fuel-asm/Cargo.toml index 5d5b7bd201..d1fcc9fb72 100644 --- a/fuel-asm/Cargo.toml +++ b/fuel-asm/Cargo.toml @@ -13,7 +13,7 @@ description = "Atomic types of the FuelVM." [dependencies] arbitrary = { version = "1.1", features = ["derive"], optional = true } bitflags = { workspace = true } -fuel-types = { workspace = true } +fuel-types = { workspace = true, default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } strum = { version = "0.24", default-features = false, features = ["derive"] } wasm-bindgen = { version = "0.2.88", optional = true } diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index dc12d92181..bca362ba82 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -11,7 +11,7 @@ repository = { workspace = true } description = "Compression and decompression of Fuel blocks for DA storage." [dependencies] -anyhow = { workspace = true } +anyhow = { version = "1.0" } fuel-derive = { workspace = true } paste = "1.0" postcard = { version = "1.0", features = ["use-std"], optional = true } @@ -19,7 +19,6 @@ serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" [dev-dependencies] -# bincode = { workspace = true } fuel-compression = { workspace = true, features = ["test-helpers"] } fuel-types = { workspace = true, features = ["serde"] } diff --git a/fuel-tx/Cargo.toml b/fuel-tx/Cargo.toml index 26ac8c2387..d74f58bb6d 100644 --- a/fuel-tx/Cargo.toml +++ b/fuel-tx/Cargo.toml @@ -11,7 +11,7 @@ repository = { workspace = true } description = "FuelVM transaction." [dependencies] -anyhow = { workspace = true, optional = true } +anyhow = { version = "1.0", optional = true } bitflags = { workspace = true } derivative = { version = "2.2.0", default-features = false, features = ["use_core"], optional = true } derive_more = { version = "0.99", default-features = false, features = ["display"] } diff --git a/fuel-types/Cargo.toml b/fuel-types/Cargo.toml index 7327d5d657..3489390a67 100644 --- a/fuel-types/Cargo.toml +++ b/fuel-types/Cargo.toml @@ -11,8 +11,8 @@ repository = { workspace = true } description = "Atomic types of the FuelVM." [dependencies] -anyhow = { workspace = true, optional = true } -fuel-compression = { workspace = true } +anyhow = { version = "1.0", optional = true } +fuel-compression = { workspace = true, optional = true } fuel-derive = { workspace = true } hex = { version = "0.4", default-features = false } rand = { version = "0.8", default-features = false, optional = true } @@ -33,7 +33,7 @@ typescript = ["wasm-bindgen"] alloc = ["hex/alloc"] random = ["rand"] serde = ["dep:serde", "alloc"] -da-compression = ["anyhow", "serde"] +da-compression = ["anyhow", "serde", "fuel-compression"] std = ["alloc", "serde?/std", "hex/std"] unsafe = [] diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index f04f840e21..052507d4c4 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -16,7 +16,7 @@ harness = false required-features = ["std"] [dependencies] -anyhow = { workspace = true, optional = true } +anyhow = { version = "1.0", optional = true } async-trait = "0.1" backtrace = { version = "0.3", optional = true } # requires debug symbols to work bitflags = { workspace = true } From 4188f371945a6944b36612aeb951884d0a74a022 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 2 Sep 2024 23:30:38 +0300 Subject: [PATCH 25/77] WIP: working towards adapting Green's trait-based approach --- fuel-compression/Cargo.toml | 11 +- fuel-compression/src/compaction.rs | 553 +++++++++++++----- fuel-compression/src/dummy_registry.rs | 203 ------- fuel-compression/src/key.rs | 84 --- fuel-compression/src/lib.rs | 28 +- fuel-compression/src/table.rs | 218 ------- fuel-crypto/src/secp256/signature.rs | 2 +- fuel-derive/src/compact.rs | 173 +++--- fuel-derive/src/lib.rs | 4 +- fuel-tx/src/lib.rs | 2 +- fuel-tx/src/transaction.rs | 2 +- fuel-tx/src/transaction/policies.rs | 4 +- fuel-tx/src/transaction/types/blob.rs | 2 +- .../types/chargeable_transaction.rs | 2 +- fuel-tx/src/transaction/types/create.rs | 2 +- fuel-tx/src/transaction/types/input.rs | 42 +- fuel-tx/src/transaction/types/input/coin.rs | 20 +- .../src/transaction/types/input/contract.rs | 2 +- .../src/transaction/types/input/message.rs | 22 +- fuel-tx/src/transaction/types/mint.rs | 4 +- fuel-tx/src/transaction/types/output.rs | 12 +- .../src/transaction/types/output/contract.rs | 2 +- fuel-tx/src/transaction/types/script.rs | 4 +- fuel-tx/src/transaction/types/storage.rs | 2 +- fuel-tx/src/transaction/types/upgrade.rs | 4 +- fuel-tx/src/transaction/types/upload.rs | 2 +- fuel-tx/src/transaction/types/utxo_id.rs | 2 +- fuel-tx/src/transaction/types/witness.rs | 4 +- fuel-tx/src/tx_pointer.rs | 2 +- fuel-types/src/array_types.rs | 4 +- fuel-types/src/compressed.rs | 443 -------------- fuel-types/src/numeric_types.rs | 2 +- fuel-vm/Cargo.toml | 1 - 33 files changed, 558 insertions(+), 1306 deletions(-) delete mode 100644 fuel-compression/src/dummy_registry.rs delete mode 100644 fuel-compression/src/table.rs delete mode 100644 fuel-types/src/compressed.rs diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index bca362ba82..00c02082e4 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -11,16 +11,13 @@ repository = { workspace = true } description = "Compression and decompression of Fuel blocks for DA storage." [dependencies] -anyhow = { version = "1.0" } +anyhow = "1.0" fuel-derive = { workspace = true } -paste = "1.0" -postcard = { version = "1.0", features = ["use-std"], optional = true } serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" [dev-dependencies] -fuel-compression = { workspace = true, features = ["test-helpers"] } +bimap = "0.6" +# postcard = { version = "1.0", features = ["use-std"] } fuel-types = { workspace = true, features = ["serde"] } - -[features] -test-helpers = ["dep:postcard"] +# fuel-compression = { path = "." } \ No newline at end of file diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 595c1775dd..0e25ff831c 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -1,3 +1,5 @@ +#![allow(missing_docs)] // TODO: add documentation + use std::{ marker::PhantomData, mem::MaybeUninit, @@ -8,53 +10,67 @@ use serde::{ Serialize, }; -use crate::{ - CompactionContext, - CountPerTable, - DecompactionContext, -}; - /// Convert data to reference-based format -pub trait Compactable { +pub trait Compressible { /// The compacted type with references - type Compact: Clone + Serialize + for<'a> Deserialize<'a>; + type Compressed: Clone + Serialize + for<'a> Deserialize<'a>; +} - /// Count max number of each key type, for upper limit of overwritten keys - fn count(&self) -> CountPerTable; +/// This type is compressable by the given compression context +pub trait CompressibleBy: Compressible +where + Ctx: ?Sized, +{ + /// Perform compression, returning the compressed data, + /// modifying the context + fn compress(&self, ctx: &mut Ctx) -> anyhow::Result; +} - /// Convert to compacted format - fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result; +/// This type is decompressable by the given decompression context +pub trait DecompressibleBy: Compressible +where + Ctx: ?Sized, + Self: Sized, +{ + /// Perform decompression, returning the original data + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result; +} - /// Convert from compacted format - fn decompact( - compact: Self::Compact, - ctx: &dyn DecompactionContext, - ) -> anyhow::Result - where - Self: Sized; +pub trait CompressionContext +where + Type: Compressible, +{ + fn compress(&mut self, value: &Type) -> anyhow::Result; +} + +pub trait DecompressionContext +where + Type: Compressible, +{ + fn decompress(&self, value: &Type::Compressed) -> anyhow::Result; } macro_rules! identity_compaction { ($t:ty) => { - impl Compactable for $t { - type Compact = Self; - - fn count(&self) -> CountPerTable { - CountPerTable::default() - } + impl Compressible for $t { + type Compressed = Self; + } - fn compact( - &self, - _ctx: &mut dyn CompactionContext, - ) -> anyhow::Result { + impl CompressibleBy for $t + where + Ctx: ?Sized, + { + fn compress(&self, _: &mut Ctx) -> anyhow::Result { Ok(*self) } + } - fn decompact( - compact: Self::Compact, - _ctx: &dyn DecompactionContext, - ) -> anyhow::Result { - Ok(compact) + impl DecompressibleBy for $t + where + Ctx: ?Sized, + { + fn decompress(c: &Self::Compressed, _: &Ctx) -> anyhow::Result { + Ok(*c) } } }; @@ -66,6 +82,31 @@ identity_compaction!(u32); identity_compaction!(u64); identity_compaction!(u128); +impl Compressible for Option +where + T: Compressible + Clone, +{ + type Compressed = Option; +} + +impl CompressibleBy for Option +where + T: CompressibleBy + Clone, +{ + fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { + Ok(self.as_ref().map(|item| item.compress(ctx)).transpose()?) + } +} + +impl DecompressibleBy for Option +where + T: DecompressibleBy + Clone, +{ + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + c.as_ref().map(|item| T::decompress(item, ctx)).transpose() + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ArrayWrapper Deserialize<'a>>( #[serde(with = "serde_big_array::BigArray")] pub [T; S], @@ -90,94 +131,95 @@ fn try_map_array Result>( Ok(tmp.map(|v| unsafe { v.assume_init() })) } -impl Compactable for [T; S] +impl Compressible for [T; S] where - T: Compactable + Clone + Serialize + for<'a> Deserialize<'a>, + T: Compressible + Clone, { - type Compact = ArrayWrapper; - - fn count(&self) -> CountPerTable { - let mut count = CountPerTable::default(); - for item in self.iter() { - count += item.count(); - } - count - } + type Compressed = ArrayWrapper; +} - fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result { +impl CompressibleBy for [T; S] +where + T: CompressibleBy + Clone, +{ + fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { Ok(ArrayWrapper(try_map_array(self.clone(), |v: T| { - v.compact(ctx) + v.compress(ctx) })?)) } +} - fn decompact( - compact: Self::Compact, - ctx: &dyn DecompactionContext, - ) -> anyhow::Result { - try_map_array(compact.0, |v: T::Compact| T::decompact(v, ctx)) +impl DecompressibleBy for [T; S] +where + T: DecompressibleBy + Clone, +{ + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + try_map_array(c.0.clone(), |v: T::Compressed| T::decompress(&v, ctx)) } } -impl Compactable for Vec +impl Compressible for Vec where - T: Compactable + Clone + Serialize + for<'a> Deserialize<'a>, + T: Compressible + Clone, { - type Compact = Vec; - - fn count(&self) -> CountPerTable { - let mut count = CountPerTable::default(); - for item in self.iter() { - count += item.count(); - } - count - } + type Compressed = Vec; +} - fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result { - self.iter().map(|item| item.compact(ctx)).collect() +impl CompressibleBy for Vec +where + T: CompressibleBy + Clone, +{ + fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { + self.iter().map(|item| item.compress(ctx)).collect() } +} - fn decompact( - compact: Self::Compact, - ctx: &dyn DecompactionContext, - ) -> anyhow::Result { - compact - .into_iter() - .map(|item| T::decompact(item, ctx)) - .collect() +impl DecompressibleBy for Vec +where + T: DecompressibleBy + Clone, +{ + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + c.into_iter().map(|item| T::decompress(item, ctx)).collect() } } -impl Compactable for PhantomData { - type Compact = (); - - fn count(&self) -> CountPerTable { - CountPerTable::default() - } +impl Compressible for PhantomData { + type Compressed = (); +} - fn compact(&self, _ctx: &mut dyn CompactionContext) -> anyhow::Result { +impl CompressibleBy for PhantomData +where + T: CompressibleBy + Clone, +{ + fn compress(&self, _: &mut Ctx) -> anyhow::Result { Ok(()) } +} - fn decompact( - _compact: Self::Compact, - _ctx: &dyn DecompactionContext, - ) -> anyhow::Result { - Ok(Self) +impl DecompressibleBy for PhantomData +where + T: DecompressibleBy + Clone, +{ + fn decompress(_: &Self::Compressed, _: &Ctx) -> anyhow::Result { + Ok(PhantomData) } } -#[cfg(test)] +#[cfg(feature = "never")] +// #[cfg(test)] mod tests { - use fuel_compression::{ - dummy_registry::DummyRegistry, - tables, - Compactable, - CompactionContext, - CountPerTable, - DecompactionContext, - Key, + use std::collections::HashMap; + + use crate::RawKey; + + use super::{ + Compressible, + CompressibleBy, + CompressionContext, + DecompressibleBy, + DecompressionContext, }; - use fuel_derive::Compact; + use fuel_derive::Compressed; use fuel_types::{ Address, AssetId, @@ -187,97 +229,284 @@ mod tests { Serialize, }; + pub struct TestCompressionContext { + pub assets: HashMap, + } + + impl Compressible for AssetId { + type Compressed = RawKey; + } + + impl CompressibleBy for AssetId + where + Ctx: CompressionContext, + { + fn compress(&self, compressor: &mut Ctx) -> anyhow::Result { + compressor.compress(self) + } + } + + impl CompressionContext for TestCompressionContext { + fn compress( + &mut self, + type_to_compact: &AssetId, + ) -> anyhow::Result<::Compressed> { + let size = self.assets.len(); + let entry = self + .assets + .entry(*type_to_compact) + .or_insert_with(|| RawKey::try_from(size as u32).unwrap()); + Ok(*entry) + } + } + #[derive(Debug, Clone, PartialEq)] struct ManualExample { - a: Address, - b: Address, + a: AssetId, + b: AssetId, c: u64, + d: [u8; 32], } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] - struct ManualExampleCompact { - a: Key, - b: Key, - c: u64, + struct ManualExampleCompressed { + a: ::Compressed, + b: ::Compressed, + c: ::Compressed, + c: <[u8; 32] as Compressible>::Compressed, } - impl Compactable for ManualExample { - type Compact = ManualExampleCompact; - - fn count(&self) -> CountPerTable { - CountPerTable::Address(2) - } + impl Compressible for ManualExample { + type Compressed = ManualExampleCompressed; + } - fn compact( - &self, - ctx: &mut dyn CompactionContext, - ) -> anyhow::Result { - let a = ctx.to_key_Address(*self.a)?; - let b = ctx.to_key_Address(*self.b)?; - Ok(ManualExampleCompact { a, b, c: self.c }) + impl CompressibleBy for ManualExample { + fn compress(&self, ctx: &mut Ctx) -> anyhow::Result + where + AssetId: CompressibleBy, + u64: CompressibleBy, + [u8; 32]: CompressibleBy, + { + Ok(ManualExampleCompressed { + a: self.a.compress(ctx)?, + b: self.b.compress(ctx)?, + c: self.c.compress(ctx)?, + d: self.d.compress(ctx)?, + }) } + } - fn decompact( - compact: Self::Compact, - ctx: &dyn DecompactionContext, - ) -> anyhow::Result { - let a = Address::from(ctx.read_Address(compact.a)?); - let b = Address::from(ctx.read_Address(compact.b)?); - Ok(Self { a, b, c: compact.c }) + impl DecompressibleBy for ManualExample { + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result + where + u64: DecompressibleBy, + { + Ok(ManualExample { + a: >::decompress(c.a, ctx)?, + b: >::decompress(c.b, ctx)?, + c: >::decompress(c.c, ctx)?, + d: <[u8; 32] as DedompressibleBy>::decompress(c.d, ctx)?, + }) } } - #[derive(Debug, Clone, PartialEq, Compact)] - struct AutomaticExample { - #[da_compress(registry = ::fuel_compression::tables::AssetId)] - a: AssetId, - #[da_compress(registry = ::fuel_compression::tables::AssetId)] - b: AssetId, - c: u32, - } + // #[derive(Debug, Clone, PartialEq, Compressed)] + // struct AutomaticExample { + // #[da_compress(registry)] + // a: AssetId, + // #[da_compress(registry)] + // b: AssetId, + // c: u32, + // } #[test] fn test_compaction_properties() { - let a = ManualExample { - a: Address::from([1u8; 32]), - b: Address::from([2u8; 32]), - c: 3, - }; - assert_eq!(a.count().Address, 2); - assert_eq!(a.count().AssetId, 0); - - let b = AutomaticExample { + let _a = ManualExample { a: AssetId::from([1u8; 32]), b: AssetId::from([2u8; 32]), c: 3, }; - assert_eq!(b.count().Address, 0); - assert_eq!(b.count().AssetId, 2); + + // let b = AutomaticExample { + // a: AssetId::from([1u8; 32]), + // b: AssetId::from([2u8; 32]), + // c: 3, + // }; + // assert_eq!(b.count().Address, 0); + // assert_eq!(b.count().AssetId, 2); } - #[test] - fn test_compaction_roundtrip_manual() { - let target = ManualExample { - a: Address::from([1u8; 32]), - b: Address::from([2u8; 32]), - c: 3, - }; - let mut registry = DummyRegistry::default(); - let (compacted, _) = registry.compact(target.clone()).unwrap(); - let decompacted = ManualExample::decompact(compacted, ®istry).unwrap(); - assert_eq!(target, decompacted); + // #[test] + // fn test_compaction_roundtrip_manual() { + // let target = ManualExample { + // a: Address::from([1u8; 32]), + // b: Address::from([2u8; 32]), + // c: 3, + // }; + // let mut registry = DummyRegistry::default(); + // let (compacted, _) = registry.compact(target.clone()).unwrap(); + // let decompacted = ManualExample::decompact(compacted, ®istry).unwrap(); + // assert_eq!(target, decompacted); + // } + + // #[test] + // fn test_compaction_roundtrip_derive() { + // let target = AutomaticExample { + // a: AssetId::from([1u8; 32]), + // b: AssetId::from([2u8; 32]), + // c: 3, + // }; + // let mut registry = DummyRegistry::default(); + // let (compacted, _) = registry.compact(target.clone()).unwrap(); + // let decompacted = AutomaticExample::decompact(compacted, ®istry).unwrap(); + // assert_eq!(target, decompacted); + // } +} + +#[cfg(test)] +mod playground { + use fuel_types::AssetId; + + use crate::RawKey; + + use super::{ + ArrayWrapper, + Compressible, + CompressibleBy, + CompressionContext, + DecompressibleBy, + DecompressionContext, + }; + + impl Compressible for AssetId { + type Compressed = RawKey; } - #[test] - fn test_compaction_roundtrip_derive() { - let target = AutomaticExample { - a: AssetId::from([1u8; 32]), - b: AssetId::from([2u8; 32]), - c: 3, - }; - let mut registry = DummyRegistry::default(); - let (compacted, _) = registry.compact(target.clone()).unwrap(); - let decompacted = AutomaticExample::decompact(compacted, ®istry).unwrap(); - assert_eq!(target, decompacted); + impl CompressibleBy for AssetId + where + C: CompressionContext, + { + fn compress(&self, compressor: &mut C) -> anyhow::Result { + compressor.compress(self) + } + } + + impl DecompressibleBy for AssetId + where + C: DecompressionContext, + { + fn decompress(c: &Self::Compressed, compressor: &C) -> anyhow::Result { + compressor.decompress(c) + } + } + + // #[derive(CompressibleBy)] + #[derive(Clone, Debug, PartialEq, Default)] + pub struct ComplexStruct { + asset_id: AssetId, + array: [u8; 32], + // #[compressible_by(skip)] + some_field: u64, + } + + // Generated code from `#[derive(CompressibleBy)]` + + #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] + pub struct CompressedComplexStruct { + asset_id: ::Compressed, + array: ArrayWrapper<32, u8>, + } + + impl Compressible for ComplexStruct { + type Compressed = CompressedComplexStruct; + } + + impl CompressibleBy for ComplexStruct + where + AssetId: CompressibleBy, + [u8; 32]: DecompressibleBy, + { + fn compress(&self, register: &mut C) -> anyhow::Result { + let asset_id = self.asset_id.compress(register)?; + let array = self.array.compress(register)?; + Ok(CompressedComplexStruct { asset_id, array }) + } + } + + impl DecompressibleBy for ComplexStruct + where + AssetId: DecompressibleBy, + [u8; 32]: DecompressibleBy, + { + fn decompress(c: &Self::Compressed, register: &C) -> anyhow::Result { + let asset_id = AssetId::decompress(&c.asset_id, register)?; + let array = + <[u8; 32] as DecompressibleBy>::decompress(&c.array, register)?; + Ok(ComplexStruct { + asset_id, + array, + ..Default::default() + }) + } + } + + mod tests { + use bimap::BiMap; + + use super::*; + + #[derive(Default)] + struct MapRegister { + assets: BiMap, + } + + impl CompressionContext for MapRegister { + fn compress(&mut self, type_to_compact: &AssetId) -> anyhow::Result { + if let Some(key) = self.assets.get_by_right(type_to_compact) { + return Ok(*key); + } + let size = self.assets.len(); + let key = RawKey::try_from(size as u32).unwrap(); + self.assets.insert(key, *type_to_compact); + Ok(key) + } + } + + impl DecompressionContext for MapRegister { + fn decompress(&self, c: &RawKey) -> anyhow::Result { + self.assets + .get_by_left(c) + .copied() + .ok_or_else(|| anyhow::anyhow!("Asset not found")) + } + } + + #[test] + fn can_register() { + let mut register = MapRegister::default(); + let complex_struct = ComplexStruct { + asset_id: [1; 32].into(), + array: [2; 32], + some_field: Default::default(), + }; + let compressed_complex = complex_struct.compress(&mut register).unwrap(); + let restored = >::decompress( + &compressed_complex, + ®ister, + ) + .unwrap(); + assert_eq!(complex_struct, restored); + + let compressed_again_complex = + complex_struct.compress(&mut register).unwrap(); + + let new_complex_struct = ComplexStruct { + asset_id: [2; 32].into(), + array: [2; 32], + some_field: 3, + }; + let compressed_new_complex = + new_complex_struct.compress(&mut register).unwrap(); + } } } diff --git a/fuel-compression/src/dummy_registry.rs b/fuel-compression/src/dummy_registry.rs deleted file mode 100644 index 894cf82638..0000000000 --- a/fuel-compression/src/dummy_registry.rs +++ /dev/null @@ -1,203 +0,0 @@ -//! Temporal registry implementation used for tests - -use std::collections::HashMap; - -use crate::{ - key::RawKey, - table::{ - access::*, - KeyPerTable, - }, - tables, - Compactable, - CompactionContext, - DecompactionContext, - Key, - Table, - TableName, -}; - -/// Temporal registry implementation used for tests. -#[derive(Default)] -pub struct DummyRegistry { - next_keys: KeyPerTable, - values: HashMap<(TableName, RawKey), Vec>, -} - -impl DummyRegistry { - /// Run the compaction for the given target, returning the compacted data. - /// Applies the changes to the registry, but also returns them for block inclusion. - pub fn compact( - &mut self, - target: C, - ) -> anyhow::Result<(C::Compact, Changes)> { - let key_limits = target.count(); - let safe_keys_start = self.next_keys.offset_by(key_limits); - - let mut ctx = DummyCompactionCtx { - start_keys: self.next_keys, - next_keys: self.next_keys, - safe_keys_start, - changes: Changes::default(), - reg: self, - }; - - let compacted = target.compact(&mut ctx)?; - let changes = ctx.changes; - for change in changes.changes.iter() { - self.values.insert((change.0, change.1), change.2.clone()); - } - Ok((compacted, changes)) - } - - fn resolve_key(&self, key: Key) -> anyhow::Result { - self.values - .get(&(::NAME, key.raw())) - .ok_or_else(|| anyhow::anyhow!("Key not found: {:?}", key)) - .and_then(|bytes| postcard::from_bytes(bytes).map_err(|e| e.into())) - } -} - -/// Changeset for the registry -#[derive(Default)] -pub struct Changes { - changes: Vec<(TableName, RawKey, Vec)>, -} -impl Changes { - fn lookup_value(&self, value: &::Type) -> Option> { - // Slow linear search. This is test-only code, so it's ok. - for change in self.changes.iter() { - if change.0 == ::NAME - && change.2 == postcard::to_stdvec(value).unwrap() - { - return Some(Key::::from_raw(change.1)); - } - } - None - } -} - -/// Compaction session for -pub struct DummyCompactionCtx<'a> { - /// The registry - reg: &'a DummyRegistry, - /// These are the keys where writing started - start_keys: KeyPerTable, - /// The next keys to use for each table - next_keys: KeyPerTable, - /// Keys in range next_keys..safe_keys_start - /// could be overwritten by the compaction, - /// and cannot be used for new values. - safe_keys_start: KeyPerTable, - changes: Changes, -} - -impl<'a> DummyCompactionCtx<'a> { - /// Convert a value to a key - /// If necessary, store the value in the changeset and allocate a new key. - fn value_to_key(&mut self, value: T::Type) -> anyhow::Result> - where - KeyPerTable: AccessCopy> + AccessMut>, - { - // Check if the value is within the current changeset - if let Some(key) = self.changes.lookup_value(&value) { - return Ok(key); - } - - // Check if the registry contains this value already. - // This is a slow linear search, but since this is test-only code, it's ok. - let encoded = postcard::to_stdvec(&value).unwrap(); - for ((table_name, raw_key), bytes) in self.reg.values.iter() { - if *table_name == ::NAME && *bytes == encoded { - let key = Key::::from_raw(*raw_key); - // Check if the value is in the possibly-overwritable range - let start: Key = self.start_keys.value(); - let end: Key = self.safe_keys_start.value(); - if !key.is_between(start, end) { - return Ok(key); - } - } - } - - // Allocate a new key for this - let key = >>::get_mut(&mut self.next_keys) - .take_next(); - self.changes.changes.push(( - ::NAME, - key.raw(), - postcard::to_stdvec(&value).unwrap(), - )); - Ok(key) - } -} - -impl<'a> CompactionContext for DummyCompactionCtx<'a> { - fn to_key_AssetId( - &mut self, - value: [u8; 32], - ) -> anyhow::Result> { - self.value_to_key(value) - } - - fn to_key_Address( - &mut self, - value: [u8; 32], - ) -> anyhow::Result> { - self.value_to_key(value) - } - - fn to_key_ContractId( - &mut self, - value: [u8; 32], - ) -> anyhow::Result> { - self.value_to_key(value) - } - - fn to_key_ScriptCode( - &mut self, - value: Vec, - ) -> anyhow::Result> { - self.value_to_key(value) - } - - fn to_key_Witness(&mut self, value: Vec) -> anyhow::Result> { - self.value_to_key(value) - } -} - -impl DecompactionContext for DummyRegistry { - fn read_AssetId( - &self, - key: Key, - ) -> anyhow::Result<::Type> { - self.resolve_key(key) - } - - fn read_Address( - &self, - key: Key, - ) -> anyhow::Result<::Type> { - self.resolve_key(key) - } - - fn read_ContractId( - &self, - key: Key, - ) -> anyhow::Result<::Type> { - self.resolve_key(key) - } - - fn read_ScriptCode( - &self, - key: Key, - ) -> anyhow::Result<::Type> { - self.resolve_key(key) - } - - fn read_Witness( - &self, - key: Key, - ) -> anyhow::Result<::Type> { - self.resolve_key(key) - } -} diff --git a/fuel-compression/src/key.rs b/fuel-compression/src/key.rs index 08c391512c..ab00d9db73 100644 --- a/fuel-compression/src/key.rs +++ b/fuel-compression/src/key.rs @@ -1,13 +1,8 @@ -use core::fmt; -use std::marker::PhantomData; - use serde::{ Deserialize, Serialize, }; -use super::Table; - /// Untyped key pointing to a registry table entry. /// The last key (all bits set) is reserved for the default value and cannot be written /// to. @@ -90,85 +85,6 @@ impl TryFrom for RawKey { } } -/// Typed key to a registry table entry. -/// The last key (all bits set) is reserved for the default value and cannot be written -/// to. -#[allow(clippy::derived_hash_with_manual_eq)] // PhantomData requires this -#[derive(Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct Key(RawKey, PhantomData); -impl Clone for Key { - fn clone(&self) -> Self { - *self - } -} -impl Copy for Key {} - -impl PartialEq> for Key { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} - -impl Key { - /// This key is reserved for the default value and cannot be written to. - pub const DEFAULT_VALUE: Self = Self(RawKey::DEFAULT_VALUE, PhantomData); - /// This is the first writable key. - pub const ZERO: Self = Self(RawKey::ZERO, PhantomData); - - /// Obtain untyped key. - pub fn raw(&self) -> RawKey { - self.0 - } - - /// Construct from untyped key. - pub fn from_raw(raw: RawKey) -> Self { - Self(raw, PhantomData) - } - - /// Wraps around at limit, i.e. one below the max/default value - pub fn add_u32(self, rhs: u32) -> Self { - Self(self.0.add_u32(rhs), PhantomData) - } - - /// Wraps around at limit, i.e. one below the max/default value - pub fn next(self) -> Self { - Self(self.0.next(), PhantomData) - } - - /// Is `self` between `start` and `end`? i.e. in the half-open logical range - /// `start`..`end`, so that wrap-around cases are handled correctly. - pub fn is_between(self, start: Self, end: Self) -> bool { - self.0.is_between(start.0, end.0) - } - - /// Increments the key by one, and returns the previous value. - /// Skips the max/default value. - pub fn take_next(&mut self) -> Self { - let result = *self; - self.0 = self.0.next(); - result - } -} - -impl TryFrom for Key { - type Error = &'static str; - - fn try_from(value: u32) -> Result { - Ok(Self(RawKey::try_from(value)?, PhantomData)) - } -} - -impl fmt::Debug for Key { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if *self == Self::DEFAULT_VALUE { - write!(f, "Key<{}>::DEFAULT_VALUE", T::NAME) - } else { - write!(f, "Key<{}>({})", T::NAME, self.0.as_u32()) - } - } -} - #[cfg(test)] mod tests { use super::RawKey; diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index ac7969b937..244b6e92d3 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -8,26 +8,14 @@ mod compaction; mod key; -mod table; -#[cfg(any(test, feature = "test-helpers"))] -pub mod dummy_registry; - -pub use compaction::Compactable; -pub use table::{ - access, - tables, - CompactionContext, - CountPerTable, - DecompactionContext, - KeyPerTable, - Table, - TableName, -}; - -pub use key::{ - Key, - RawKey, +pub use compaction::{ + Compressible, + CompressibleBy, + CompressionContext, + DecompressibleBy, + DecompressionContext, }; +pub use key::RawKey; -pub use fuel_derive::Compact; +pub use fuel_derive::Compressed; diff --git a/fuel-compression/src/table.rs b/fuel-compression/src/table.rs deleted file mode 100644 index 391a080ddd..0000000000 --- a/fuel-compression/src/table.rs +++ /dev/null @@ -1,218 +0,0 @@ -use serde::{ - Deserialize, - Serialize, -}; - -use crate::{ - Key, - RawKey, -}; - -mod _private { - pub trait Seal {} -} - -/// Static name of a table -pub type TableName = &'static str; - -/// Table in the registry -pub trait Table: _private::Seal { - /// Unique name of the table - const NAME: TableName; - - /// A `CountPerTable` for this table - fn count(n: usize) -> CountPerTable; - - /// The type stored in the table - type Type: PartialEq + Default + Serialize + for<'de> Deserialize<'de>; -} - -/// Traits for accessing `*PerTable` using the table type -pub mod access { - /// Copy value for the give table - pub trait AccessCopy { - /// Copy value for the give table - fn value(&self) -> V; - } - - /// Get reference to the value for the given table - pub trait AccessRef { - /// Get reference to the value for the given table - fn get(&self) -> &V; - } - - /// Get mutable reference to the value for the given table - pub trait AccessMut { - /// Get mutable reference to the value for the given table - fn get_mut(&mut self) -> &mut V; - } -} - -macro_rules! tables { - ($($name:ident: $ty:ty),*$(,)?) => { paste::paste! { - /// Marker struct for each table type - pub mod tables { - $( - /// Specifies the table to use for a given key. - /// The data is separated to tables based on the data type being stored. - #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] - pub struct $name; - - impl super::_private::Seal for $name {} - impl super::Table for $name { - const NAME: &'static str = stringify!($name); - fn count(n: usize) -> super::CountPerTable { - super::CountPerTable::$name(n) - } - type Type = $ty; - } - - impl $name { - /// Calls the `to_key_*` method for this table on the context - pub fn to_key(value: $ty, ctx: &mut dyn super::CompactionContext) -> anyhow::Result> { - ctx.[](value) - } - - /// Calls the `read_*` method for this table on the context - pub fn read(key: super::Key<$name>, ctx: &dyn super::DecompactionContext) -> anyhow::Result<$ty> { - ctx.[](key) - } - } - )* - } - - /// Context for compaction, i.e. converting data to reference-based format. - /// The context is used to aggreage changes to the registry. - /// A new context should be created for each compaction "session", - /// typically a blockchain block. - #[allow(non_snake_case)] // The field names match table type names eactly - pub trait CompactionContext { - $( - /// Store a value to the changeset and return a short reference key to it. - /// If the value already exists in the registry and will not be overwritten, - /// the existing key can be returned instead. - fn [](&mut self, value: $ty) -> anyhow::Result>; - )* - } - - /// Context for compaction, i.e. converting data to reference-based format - #[allow(non_snake_case)] // The field names match table type names eactly - pub trait DecompactionContext { - $( - /// Read a value from the registry based on the key. - fn [](&self, key: Key) -> anyhow::Result<::Type>; - )* - } - - /// One counter per table - #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] - #[allow(non_snake_case)] // The field names match table type names eactly - #[allow(missing_docs)] // Makes no sense to document the fields - #[non_exhaustive] - pub struct CountPerTable { - $(pub $name: usize),* - } - - impl CountPerTable {$( - /// Custom constructor per table - #[allow(non_snake_case)] // The field names match table type names eactly - pub fn $name(value: usize) -> Self { - Self { - $name: value, - ..Self::default() - } - } - )*} - - $( - impl access::AccessCopy for CountPerTable { - fn value(&self) -> usize { - self.$name - } - } - )* - - impl core::ops::Add for CountPerTable { - type Output = Self; - - fn add(self, rhs: CountPerTable) -> Self::Output { - Self { - $($name: self.$name + rhs.$name),* - } - } - } - - impl core::ops::AddAssign for CountPerTable { - fn add_assign(&mut self, rhs: CountPerTable) { - $(self.$name += rhs.$name);* - } - } - - /// One key value per table - #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] - #[allow(non_snake_case)] // The field names match table type names eactly - #[allow(missing_docs)] // Makes no sense to document the fields - #[non_exhaustive] - pub struct KeyPerTable { - $(pub $name: Key),* - } - - impl Default for KeyPerTable { - fn default() -> Self { - Self { - $($name: Key::ZERO,)* - } - } - } - - impl KeyPerTable { - /// Generate keys for each table using a function that's called for each table. - /// Since type erasure is required, the function takes `TableName` adn returns `RawKey`. - pub fn from_fn RawKey>(f: F) -> Self { - Self { - $($name: Key::from_raw(f(::NAME)),)* - } - } - } - - $( - impl access::AccessCopy> for KeyPerTable { - fn value(&self) -> Key { - self.$name - } - } - impl access::AccessRef> for KeyPerTable { - fn get(&self) -> &Key { - &self.$name - } - } - impl access::AccessMut> for KeyPerTable { - fn get_mut(&mut self) -> &mut Key { - &mut self.$name - } - } - )* - - impl KeyPerTable { - /// Used to add together keys and counts to deterimine possible overwrite range. - /// Panics if the keys count cannot fit into `u32`. - pub fn offset_by(&self, counts: CountPerTable) -> KeyPerTable { - KeyPerTable { - $( - $name: self.$name.add_u32(counts.$name.try_into() - .expect("Count too large. Shouldn't happen as we control inputs here.") - ), - )* - } - } - } - }}; -} - -tables!( - AssetId: [u8; 32], - Address: [u8; 32], - ContractId: [u8; 32], - ScriptCode: Vec, - Witness: Vec, -); diff --git a/fuel-crypto/src/secp256/signature.rs b/fuel-crypto/src/secp256/signature.rs index 132ad55a12..ff51e65052 100644 --- a/fuel-crypto/src/secp256/signature.rs +++ b/fuel-crypto/src/secp256/signature.rs @@ -14,7 +14,7 @@ use core::{ str, }; -/// Compact-form Secp256k1 signature. +/// Compressed-form Secp256k1 signature. #[derive(Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compact.rs index addd92b8f0..1937b9e178 100644 --- a/fuel-derive/src/compact.rs +++ b/fuel-derive/src/compact.rs @@ -95,15 +95,13 @@ impl StructureAttrs { pub enum FieldAttrs { /// Skipped when compacting, and must be reconstructed when decompacting. Skip, - /// Compacted recursively. + /// Compresseded recursively. Normal, /// This value is compacted into a registry lookup. - Registry(syn::Path), + Registry, } impl FieldAttrs { pub fn parse(attrs: &[syn::Attribute]) -> Self { - let registry_path = syn::parse2::(quote! {registry}).unwrap(); - let mut result = Self::Normal; for attr in attrs { if attr.style != syn::AttrStyle::Outer { @@ -120,15 +118,9 @@ impl FieldAttrs { if ident == "skip" { result = Self::Skip; continue; - } - } else if let Ok(kv) = - syn::parse2::(ml.tokens.clone()) - { - if kv.path == registry_path { - if let syn::Expr::Path(p) = kv.value { - result = Self::Registry(p.path); - continue; - } + } else if ident == "registry" { + result = Self::Registry; + continue; } } panic!("Invalid attribute: {}", ml.tokens); @@ -151,7 +143,7 @@ fn field_defs(fields: &syn::Fields) -> TokenStream2 { FieldAttrs::Normal => { let ty = &field.ty; let cty = quote! { - <#ty as ::fuel_compression::Compactable>::Compact + <#ty as ::fuel_compression::Compressible>::Compressed }; if let Some(fname) = field.ident.as_ref() { quote! { #fname: #cty, } @@ -159,14 +151,11 @@ fn field_defs(fields: &syn::Fields) -> TokenStream2 { quote! { #cty, } } } - FieldAttrs::Registry(registry) => { - let cty = quote! { - ::fuel_compression::Key<#registry> - }; + FieldAttrs::Registry => { if let Some(fname) = field.ident.as_ref() { - quote! { #fname: #cty, } + quote! { #fname: ::fuel_compression::RawKey, } } else { - quote! { #cty, } + quote! { ::fuel_compression::RawKey, } } } }); @@ -197,20 +186,12 @@ fn construct_compact( FieldAttrs::Skip => quote! {}, FieldAttrs::Normal => { quote! { - let #cname = <#ty as Compactable>::compact(&#binding, ctx)?; + let #cname = <#ty as ::fuel_compression::CompressibleBy<_>>::compress(&#binding, ctx)?; } } - FieldAttrs::Registry(registry) => { - let cty = quote! { - Key< - #registry - > - }; + FieldAttrs::Registry => { quote! { - let #cname: #cty = #registry::to_key( - <#registry as Table>::Type::from(#binding.clone()), - ctx, - )?; + let #cname = <#ty as ::fuel_compression::CompressibleBy<_>>::compress(&#binding, ctx)?; } } } @@ -265,16 +246,12 @@ fn construct_decompact( }, FieldAttrs::Normal => { quote! { - let #cname = <#ty as Compactable>::decompact(#binding, ctx)?; + let #cname = <#ty as ::fuel_compression::DecompressibleBy<_>>::decompress(#binding, ctx)?; } } - FieldAttrs::Registry(registry) => { + FieldAttrs::Registry => { quote! { - let raw: <#registry as Table>::Type = #registry::read( - #binding, - ctx, - )?; - let #cname = raw.into(); + let #cname = <#ty as ::fuel_compression::DecompressibleBy<_>>::decompress(#binding, ctx)?; } } } @@ -306,33 +283,6 @@ fn construct_decompact( } } -// Sum of Compactable::count() of all fields. -fn sum_counts(variant: &synstructure::VariantInfo<'_>) -> TokenStream2 { - variant - .bindings() - .iter() - .map(|binding| { - let attrs = FieldAttrs::parse(&binding.ast().attrs); - let ty = &binding.ast().ty; - - match attrs { - FieldAttrs::Skip => quote! { CountPerTable::default() }, - FieldAttrs::Normal => { - quote! { <#ty as Compactable>::count(&#binding) } - } - FieldAttrs::Registry(registry) => { - quote! { - #registry::count(1) - } - } - } - }) - .fold( - quote! { CountPerTable::default() }, - |acc, x| quote! { #acc + #x }, - ) -} - /// Generate a match arm for each variant of the compacted structure /// using the given function to generate the pattern body. fn each_variant_compact) -> TokenStream2>( @@ -364,7 +314,7 @@ fn each_variant_compact) -> TokenStream2 .collect() } -/// Derives `Compact` trait for the given `struct` or `enum`. +/// Derives `Compressed` trait for the given `struct` or `enum`. pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { s.add_bounds(synstructure::AddBounds::None) .underscore_const(true); @@ -375,7 +325,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { }; let name = &s.ast().ident; - let compact_name = format_ident!("Compact{}", name); + let compact_name = format_ident!("Compressed{}", name); let mut g = s.ast().generics.clone(); let mut w_structure = g.where_clause.take(); @@ -395,13 +345,11 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { } for p in bound { let id = syn::Ident::new(p, Span::call_site()); - w_structure - .as_mut() - .unwrap() - .predicates - .push(syn::parse_quote! { #id: ::fuel_compression::Compactable }); + w_structure.as_mut().unwrap().predicates.push( + syn::parse_quote! { #id: ::fuel_compression::Compressible }, + ); w_impl.as_mut().unwrap().predicates.push( - syn::parse_quote! { for<'de> #id: ::fuel_compression::Compactable + serde::Serialize + serde::Deserialize<'de> + Clone }, + syn::parse_quote! { for<'de> #id: ::fuel_compression::Compressible + serde::Serialize + serde::Deserialize<'de> + Clone }, ); } } @@ -420,6 +368,56 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { } } + let mut w_impl_field_bounds_compress = w_impl.clone(); + for variant in s.variants() { + for field in variant.ast().fields.iter() { + let ty = &field.ty; + let attrs = FieldAttrs::parse(&field.attrs); + if matches!(attrs, FieldAttrs::Skip) { + continue; + } + + if w_impl_field_bounds_compress.is_none() { + w_impl_field_bounds_compress = Some(syn::WhereClause { + where_token: syn::Token![where](proc_macro2::Span::call_site()), + predicates: Default::default(), + }); + } + + w_impl_field_bounds_compress + .as_mut() + .unwrap() + .predicates + .push(syn::parse_quote! { #ty: ::fuel_compression::CompressibleBy }); + } + } + + let mut w_impl_field_bounds_decompress = w_impl.clone(); + for variant in s.variants() { + for field in variant.ast().fields.iter() { + let ty = &field.ty; + let attrs = FieldAttrs::parse(&field.attrs); + if matches!(attrs, FieldAttrs::Skip) { + continue; + } + + if w_impl_field_bounds_decompress.is_none() { + w_impl_field_bounds_decompress = Some(syn::WhereClause { + where_token: syn::Token![where](proc_macro2::Span::call_site()), + predicates: Default::default(), + }); + } + + w_impl_field_bounds_decompress + .as_mut() + .unwrap() + .predicates + .push( + syn::parse_quote! { #ty: ::fuel_compression::DecompressibleBy }, + ); + } + } + let def = match &s.ast().data { syn::Data::Struct(v) => { let variant: &synstructure::VariantInfo = &s.variants()[0]; @@ -431,7 +429,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { }; quote! { #[derive(Clone, serde::Serialize, serde::Deserialize)] - #[doc = concat!("Compacted version of `", stringify!(#name), "`.")] + #[doc = concat!("Compresseded version of `", stringify!(#name), "`.")] pub struct #compact_name #g #w_structure #defs #semi } } @@ -450,14 +448,13 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { quote! { #[derive(Clone, serde::Serialize, serde::Deserialize)] - #[doc = concat!("Compacted version of `", stringify!(#name), "`.")] + #[doc = concat!("Compresseded version of `", stringify!(#name), "`.")] pub enum #compact_name #g #w_structure { #variant_defs } } } syn::Data::Union(_) => panic!("unions are not supported"), }; - let count_per_variant = s.each_variant(sum_counts); let construct_per_variant = s.each_variant(|variant| { let vname = variant.ast().ident.clone(); let construct = match &s.ast().data { @@ -480,24 +477,18 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { }); let impls = s.gen_impl(quote! { - use ::fuel_compression::{ - tables, Table, Key, Compactable, CountPerTable, - CompactionContext, DecompactionContext - }; - - gen impl Compactable for @Self #w_impl { - type Compact = #compact_name #g; - - fn count(&self) -> CountPerTable { - match self { #count_per_variant } - } + gen impl ::fuel_compression::Compressible for @Self #w_impl { + type Compressed = #compact_name #g; + } - fn compact(&self, ctx: &mut dyn CompactionContext) -> anyhow::Result { + gen impl ::fuel_compression::CompressibleBy for @Self #w_impl_field_bounds_compress { + fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { Ok(match self { #construct_per_variant }) } - - fn decompact(compact: Self::Compact, ctx: &dyn DecompactionContext) -> anyhow::Result { - Ok(match compact { #decompact_per_variant }) + } + gen impl ::fuel_compression::DecompressibleBy for @Self #w_impl_field_bounds_decompress { + fn decompress(compressed: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + Ok(match compressed { #decompact_per_variant }) } } }); diff --git a/fuel-derive/src/lib.rs b/fuel-derive/src/lib.rs index 023d0f1121..7b261fa320 100644 --- a/fuel-derive/src/lib.rs +++ b/fuel-derive/src/lib.rs @@ -32,7 +32,7 @@ synstructure::decl_derive!( serialize_derive ); synstructure::decl_derive!( - [Compact, attributes(da_compress)] => - /// Derives `Compact` trait for the given `struct` or `enum`. + [Compressed, attributes(da_compress)] => + /// Derives `Compressed` trait for the given `struct` or `enum`. compact_derive ); diff --git a/fuel-tx/src/lib.rs b/fuel-tx/src/lib.rs index 4b6e2292ef..2886021fff 100644 --- a/fuel-tx/src/lib.rs +++ b/fuel-tx/src/lib.rs @@ -126,7 +126,7 @@ pub use transaction::{ }; #[cfg(feature = "da-compression")] -pub use transaction::CompactTransaction; +pub use transaction::CompressedTransaction; pub use transaction::{ PrepareSign, diff --git a/fuel-tx/src/transaction.rs b/fuel-tx/src/transaction.rs index a396e22b7a..026a822501 100644 --- a/fuel-tx/src/transaction.rs +++ b/fuel-tx/src/transaction.rs @@ -94,7 +94,7 @@ pub type TxId = Bytes32; /// The fuel transaction entity . #[derive(Debug, Clone, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[allow(clippy::large_enum_variant)] pub enum Transaction { Script(Script), diff --git a/fuel-tx/src/transaction/policies.rs b/fuel-tx/src/transaction/policies.rs index f8ef19fecb..63437e87b6 100644 --- a/fuel-tx/src/transaction/policies.rs +++ b/fuel-tx/src/transaction/policies.rs @@ -23,7 +23,7 @@ use rand::{ /// See https://github.com/FuelLabs/fuel-specs/blob/master/src/tx-format/policy.md#policy #[derive(Clone, Copy, Default, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] pub struct PoliciesBits(u32); bitflags::bitflags! { @@ -85,7 +85,7 @@ pub const POLICIES_NUMBER: usize = PoliciesBits::all().bits().count_ones() as us /// Container for managing policies. #[derive(Clone, Copy, Default, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] pub struct Policies { /// A bitmask that indicates what policies are set. diff --git a/fuel-tx/src/transaction/types/blob.rs b/fuel-tx/src/transaction/types/blob.rs index 888a11854d..21019ade89 100644 --- a/fuel-tx/src/transaction/types/blob.rs +++ b/fuel-tx/src/transaction/types/blob.rs @@ -47,7 +47,7 @@ pub struct BlobMetadata; /// The body of the [`Blob`] transaction. #[derive(Clone, Default, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Blob)] #[derivative(Eq, PartialEq, Hash, Debug)] diff --git a/fuel-tx/src/transaction/types/chargeable_transaction.rs b/fuel-tx/src/transaction/types/chargeable_transaction.rs index c44b70c171..8b64dd3471 100644 --- a/fuel-tx/src/transaction/types/chargeable_transaction.rs +++ b/fuel-tx/src/transaction/types/chargeable_transaction.rs @@ -45,7 +45,7 @@ pub struct ChargeableMetadata { #[derive(Clone, Derivative)] #[derivative(Eq, PartialEq, Hash, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[cfg_attr(feature = "da-compression", da_compress(bound(Body)))] #[cfg_attr(feature = "da-compression", da_compress(discard(MetadataBody)))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] diff --git a/fuel-tx/src/transaction/types/create.rs b/fuel-tx/src/transaction/types/create.rs index 7bff382ee6..8998cfe707 100644 --- a/fuel-tx/src/transaction/types/create.rs +++ b/fuel-tx/src/transaction/types/create.rs @@ -71,7 +71,7 @@ impl CreateMetadata { #[derive(Default, Debug, Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Create)] #[derivative(Eq, PartialEq, Hash)] diff --git a/fuel-tx/src/transaction/types/input.rs b/fuel-tx/src/transaction/types/input.rs index e4fb16cd46..099ff5cb0f 100644 --- a/fuel-tx/src/transaction/types/input.rs +++ b/fuel-tx/src/transaction/types/input.rs @@ -14,7 +14,7 @@ use core::{ fmt::Formatter, }; #[cfg(feature = "da-compression")] -use fuel_compression::Compactable; +use fuel_compression::Compressible; use fuel_crypto::{ Hasher, PublicKey, @@ -102,29 +102,25 @@ impl Deserialize for Empty { } #[cfg(feature = "da-compression")] -impl Compactable for Empty +impl Compressible for Empty where - T: Compactable, + T: Compressible, { - type Compact = (); - - fn count(&self) -> fuel_compression::CountPerTable { - Default::default() - } - - fn compact( - &self, - _: &mut dyn fuel_compression::CompactionContext, - ) -> anyhow::Result { - Ok(()) - } - - fn decompact( - _: Self::Compact, - _: &dyn fuel_compression::DecompactionContext, - ) -> anyhow::Result { - Ok(Self(Default::default())) - } + type Compressed = (); + + // fn compact( + // &self, + // _: &mut dyn fuel_compression::CompressionContext, + // ) -> anyhow::Result { + // Ok(()) + // } + + // fn decompact( + // _: Self::Compressed, + // _: &dyn fuel_compression::DecompactionContext, + // ) -> anyhow::Result { + // Ok(Self(Default::default())) + // } } impl AsFieldFmt for Empty { @@ -225,7 +221,7 @@ where #[derive(Debug, Clone, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] pub enum Input { CoinSigned(CoinSigned), CoinPredicate(CoinPredicate), diff --git a/fuel-tx/src/transaction/types/input/coin.rs b/fuel-tx/src/transaction/types/input/coin.rs index ede356b924..65a46a61ee 100644 --- a/fuel-tx/src/transaction/types/input/coin.rs +++ b/fuel-tx/src/transaction/types/input/coin.rs @@ -12,7 +12,7 @@ use crate::{ use alloc::vec::Vec; use derivative::Derivative; #[cfg(feature = "da-compression")] -use fuel_compression::Compactable; +use fuel_compression::Compressible; use fuel_types::{ Address, AssetId, @@ -34,10 +34,10 @@ mod private { /// Specifies the coin based on the usage context. See [`Coin`]. #[cfg(feature = "da-compression")] pub trait CoinSpecification: private::Seal { - type Witness: AsField + Compactable + Clone; - type Predicate: AsField> + Compactable + Clone; - type PredicateData: AsField> + Compactable + Clone; - type PredicateGasUsed: AsField + Compactable + Clone; + type Witness: AsField + Compressible + Clone; + type Predicate: AsField> + Compressible + Clone; + type PredicateData: AsField> + Compressible + Clone; + type PredicateGasUsed: AsField + Compressible + Clone; } #[cfg(not(feature = "da-compression"))] pub trait CoinSpecification: private::Seal { @@ -49,7 +49,7 @@ pub trait CoinSpecification: private::Seal { #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] pub struct Signed; impl CoinSpecification for Signed { @@ -61,7 +61,7 @@ impl CoinSpecification for Signed { #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] pub struct Predicate; impl CoinSpecification for Predicate { @@ -107,17 +107,17 @@ impl CoinSpecification for Full { #[derive(Default, Derivative, Clone, PartialEq, Eq, Hash)] #[derivative(Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] pub struct Coin where Specification: CoinSpecification + Clone, { pub utxo_id: UtxoId, - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] pub owner: Address, pub amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] pub asset_id: AssetId, pub tx_pointer: TxPointer, #[derivative(Debug(format_with = "fmt_as_field"))] diff --git a/fuel-tx/src/transaction/types/input/contract.rs b/fuel-tx/src/transaction/types/input/contract.rs index 56f70826ab..97b6ff1ce0 100644 --- a/fuel-tx/src/transaction/types/input/contract.rs +++ b/fuel-tx/src/transaction/types/input/contract.rs @@ -14,7 +14,7 @@ use fuel_types::{ /// the `fuel-vm`. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen(js_name = InputContract))] pub struct Contract { diff --git a/fuel-tx/src/transaction/types/input/message.rs b/fuel-tx/src/transaction/types/input/message.rs index cec7f1d4eb..cf549ff4da 100644 --- a/fuel-tx/src/transaction/types/input/message.rs +++ b/fuel-tx/src/transaction/types/input/message.rs @@ -5,7 +5,7 @@ use crate::{ use alloc::vec::Vec; use derivative::Derivative; #[cfg(feature = "da-compression")] -use fuel_compression::Compactable; +use fuel_compression::Compressible; use fuel_types::{ Address, MessageId, @@ -35,28 +35,28 @@ mod private { #[cfg(feature = "da-compression")] pub trait MessageSpecification: private::Seal { type Data: AsField> - + Compactable + + Compressible + Clone + serde::Serialize + for<'a> serde::Deserialize<'a>; type Predicate: AsField> - + Compactable + + Compressible + Clone + serde::Serialize + for<'a> serde::Deserialize<'a>; type PredicateData: AsField> - + Compactable + + Compressible + Clone + serde::Serialize + for<'a> serde::Deserialize<'a>; type PredicateGasUsed: AsField - + Compactable + + Compressible + Clone + serde::Serialize + for<'a> serde::Deserialize<'a> + Default; type Witness: AsField - + Compactable + + Compressible + Clone + serde::Serialize + for<'a> serde::Deserialize<'a>; @@ -82,14 +82,14 @@ pub mod specifications { /// `witnesses` vector of the [`crate::Transaction`]. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] pub struct Signed; /// The type means that the message is not signed, and the `owner` is a `predicate` /// bytecode. The merkle root from the `predicate` should be equal to the `owner`. #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] pub struct Predicate; /// The retrayable message metadata. It is a message that can't be used as a coin to @@ -177,17 +177,17 @@ pub mod specifications { #[derive(Default, Derivative, Clone, PartialEq, Eq, Hash)] #[derivative(Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] pub struct Message where Specification: MessageSpecification + Clone, { /// The sender from the L1 chain. - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] pub sender: Address, /// The receiver on the `Fuel` chain. - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] pub recipient: Address, pub amount: Word, pub nonce: Nonce, diff --git a/fuel-tx/src/transaction/types/mint.rs b/fuel-tx/src/transaction/types/mint.rs index 26948d2cff..6dea93860e 100644 --- a/fuel-tx/src/transaction/types/mint.rs +++ b/fuel-tx/src/transaction/types/mint.rs @@ -49,7 +49,7 @@ impl MintMetadata { /// by it. #[derive(Default, Debug, Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Mint)] #[derivative(Eq, PartialEq, Hash)] @@ -64,7 +64,7 @@ pub struct Mint { /// The amount of funds minted. pub(crate) mint_amount: Word, /// The asset IDs corresponding to the minted amount. - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] pub(crate) mint_asset_id: AssetId, /// Gas Price used for current block pub(crate) gas_price: Word, diff --git a/fuel-tx/src/transaction/types/output.rs b/fuel-tx/src/transaction/types/output.rs index 6eea7aed53..93b81f0dc2 100644 --- a/fuel-tx/src/transaction/types/output.rs +++ b/fuel-tx/src/transaction/types/output.rs @@ -21,25 +21,25 @@ pub use repr::OutputRepr; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(canonical::Deserialize, canonical::Serialize)] pub enum Output { Coin { - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] to: Address, amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] asset_id: AssetId, }, Contract(Contract), Change { - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Address))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] to: Address, #[cfg_attr(feature = "da-compression", da_compress(skip))] amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::AssetId))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] asset_id: AssetId, }, @@ -53,7 +53,7 @@ pub enum Output { }, ContractCreated { - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::ContractId))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] contract_id: ContractId, state_root: Bytes32, }, diff --git a/fuel-tx/src/transaction/types/output/contract.rs b/fuel-tx/src/transaction/types/output/contract.rs index ae3b5bf619..0b23aef20a 100644 --- a/fuel-tx/src/transaction/types/output/contract.rs +++ b/fuel-tx/src/transaction/types/output/contract.rs @@ -7,7 +7,7 @@ use fuel_types::Bytes32; /// the `fuel-vm`. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen(js_name = OutputContract))] pub struct Contract { diff --git a/fuel-tx/src/transaction/types/script.rs b/fuel-tx/src/transaction/types/script.rs index 0121bdfb42..d3af17d8c5 100644 --- a/fuel-tx/src/transaction/types/script.rs +++ b/fuel-tx/src/transaction/types/script.rs @@ -48,14 +48,14 @@ pub struct ScriptMetadata { #[derive(Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[canonical(prefix = TransactionRepr::Script)] #[derivative(Eq, PartialEq, Hash, Debug)] pub struct ScriptBody { pub(crate) script_gas_limit: Word, pub(crate) receipts_root: Bytes32, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::ScriptCode))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] pub(crate) script: Vec, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] pub(crate) script_data: Vec, diff --git a/fuel-tx/src/transaction/types/storage.rs b/fuel-tx/src/transaction/types/storage.rs index 4a72959baf..3a4ee88c36 100644 --- a/fuel-tx/src/transaction/types/storage.rs +++ b/fuel-tx/src/transaction/types/storage.rs @@ -19,7 +19,7 @@ use core::cmp::Ordering; #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derive(Deserialize, Serialize)] pub struct StorageSlot { diff --git a/fuel-tx/src/transaction/types/upgrade.rs b/fuel-tx/src/transaction/types/upgrade.rs index 5dc445b2e9..652f88cf62 100644 --- a/fuel-tx/src/transaction/types/upgrade.rs +++ b/fuel-tx/src/transaction/types/upgrade.rs @@ -96,7 +96,7 @@ impl UpgradeMetadata { /// transaction. #[derive(Copy, Clone, Derivative, strum_macros::EnumCount)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[derivative(Eq, PartialEq, Hash, Debug)] pub enum UpgradePurpose { @@ -123,7 +123,7 @@ pub enum UpgradePurpose { /// The body of the [`Upgrade`] transaction. #[derive(Clone, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Upgrade)] #[derivative(Eq, PartialEq, Hash, Debug)] diff --git a/fuel-tx/src/transaction/types/upload.rs b/fuel-tx/src/transaction/types/upload.rs index 067b9eb55d..cd3ba81536 100644 --- a/fuel-tx/src/transaction/types/upload.rs +++ b/fuel-tx/src/transaction/types/upload.rs @@ -39,7 +39,7 @@ pub struct UploadMetadata; /// The body of the [`Upload`] transaction. #[derive(Clone, Default, Derivative)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] #[canonical(prefix = TransactionRepr::Upload)] #[derivative(Eq, PartialEq, Hash, Debug)] diff --git a/fuel-tx/src/transaction/types/utxo_id.rs b/fuel-tx/src/transaction/types/utxo_id.rs index 3af2cb2e58..10cfb56dc4 100644 --- a/fuel-tx/src/transaction/types/utxo_id.rs +++ b/fuel-tx/src/transaction/types/utxo_id.rs @@ -20,7 +20,7 @@ use rand::{ #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] pub struct UtxoId { /// transaction id diff --git a/fuel-tx/src/transaction/types/witness.rs b/fuel-tx/src/transaction/types/witness.rs index 9d0761b0c3..7ce60edba5 100644 --- a/fuel-tx/src/transaction/types/witness.rs +++ b/fuel-tx/src/transaction/types/witness.rs @@ -26,11 +26,11 @@ use rand::{ #[derivative(Debug)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] pub struct Witness { #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] - #[cfg_attr(feature = "da-compression", da_compress(registry = ::fuel_compression::tables::Witness))] + #[cfg_attr(feature = "da-compression", da_compress(registry))] data: Vec, } diff --git a/fuel-tx/src/tx_pointer.rs b/fuel-tx/src/tx_pointer.rs index 73fdeb052c..b36b4cef82 100644 --- a/fuel-tx/src/tx_pointer.rs +++ b/fuel-tx/src/tx_pointer.rs @@ -26,7 +26,7 @@ use rand::{ #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive(Deserialize, Serialize)] pub struct TxPointer { /// Block height diff --git a/fuel-types/src/array_types.rs b/fuel-types/src/array_types.rs index 7a4e2d68fb..51a0d06ac6 100644 --- a/fuel-types/src/array_types.rs +++ b/fuel-types/src/array_types.rs @@ -33,7 +33,7 @@ macro_rules! key { /// FuelVM atomic array type. #[repr(transparent)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] - #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, )] @@ -56,7 +56,7 @@ macro_rules! key_with_big_array { #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] - #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, )] diff --git a/fuel-types/src/compressed.rs b/fuel-types/src/compressed.rs deleted file mode 100644 index 85910949fa..0000000000 --- a/fuel-types/src/compressed.rs +++ /dev/null @@ -1,443 +0,0 @@ -//! Compressed encoding and decoding of Fuel types. -//! -//! This is an extension of the canonical encoding. - -#[cfg(feature = "alloc")] -use alloc::vec::Vec; -use core::fmt; - -use core::mem::MaybeUninit; -pub use fuel_derive::{ - Deserialize, - Serialize, -}; - -/// Allows serialize the type into the `Output`. -pub trait SerializeCompact { - /// Size of the static part of the serialized object, in bytes. - fn size_static(&self) -> usize; - - /// Size of the dynamic part, in bytes. - fn size_dynamic(&self) -> usize; - - /// Total size of the serialized object, in bytes. - fn size(&self) -> usize { - self.size_static().saturating_add(self.size_dynamic()) - } - - /// Encodes `Self` into the `buffer`. - /// - /// It is better to not implement this function directly, instead implement - /// `encode_static` and `encode_dynamic`. - fn encode(&self, buffer: &mut O) -> Result<(), Error> { - self.encode_static(buffer)?; - self.encode_dynamic(buffer) - } - - /// Encodes staticly-sized part of `Self`. - fn encode_static(&self, buffer: &mut O) -> Result<(), Error>; - - /// Encodes dynamically-sized part of `Self`. - /// The default implementation does nothing. Dynamically-sized contains should - /// override this. - fn encode_dynamic(&self, _buffer: &mut O) -> Result<(), Error> { - Ok(()) - } - - /// Encodes `Self` into bytes vector. Required known size. - #[cfg(feature = "alloc")] - fn to_bytes(&self) -> Vec { - let mut vec = Vec::with_capacity(self.size()); - self.encode(&mut vec).expect("Unable to encode self"); - vec - } -} - - -/// Allows deserialize the type from the `Input`. -pub trait DeserializeCompact: Sized { - /// Decodes `Self` from the `buffer`. - /// - /// It is better to not implement this function directly, instead implement - /// `decode_static` and `decode_dynamic`. - fn decode(buffer: &mut I) -> Result { - let mut object = Self::decode_static(buffer)?; - object.decode_dynamic(buffer)?; - Ok(object) - } - - /// Decodes static part of `Self` from the `buffer`. - fn decode_static(buffer: &mut I) -> Result; - - /// Decodes dynamic part of the information from the `buffer` to fill `Self`. - /// The default implementation does nothing. Dynamically-sized contains should - /// override this. - fn decode_dynamic( - &mut self, - _buffer: &mut I, - ) -> Result<(), Error> { - Ok(()) - } - - /// Helper method for deserializing `Self` from bytes. - fn from_bytes(mut buffer: &[u8]) -> Result { - Self::decode(&mut buffer) - } -} - - -macro_rules! impl_for_primitives { - ($t:ident, $unpadded:literal) => { - impl SerializeCompact for $t { - #[inline(always)] - fn size_static(&self) -> usize { - aligned_size(::core::mem::size_of::<$t>()) - } - - #[inline(always)] - fn size_dynamic(&self) -> usize { - 0 - } - - #[inline(always)] - fn encode_static( - &self, - buffer: &mut O, - ) -> Result<(), Error> { - // Primitive types are zero-padded on left side to a 8-byte boundary. - // The resulting value is always well-aligned. - let bytes = <$t>::to_be_bytes(*self); - for _ in 0..alignment_bytes(bytes.len()) { - // Zero-pad - buffer.push_byte(0)?; - } - buffer.write(bytes.as_ref())?; - Ok(()) - } - } - - impl DeserializeCompact for $t { - fn decode_static(buffer: &mut I) -> Result { - let mut asset = [0u8; ::core::mem::size_of::<$t>()]; - buffer.skip(alignment_bytes(asset.len()))?; // Skip zero-padding - buffer.read(asset.as_mut())?; - Ok(<$t>::from_be_bytes(asset)) - } - } - }; -} - -impl_for_primitives!(u8, true); -impl_for_primitives!(u16, false); -impl_for_primitives!(u32, false); -impl_for_primitives!(usize, false); -impl_for_primitives!(u64, false); -impl_for_primitives!(u128, false); - -// Empty tuple `()`, i.e. the unit type takes up no space. -impl Serialize for () { - fn size_static(&self) -> usize { - 0 - } - - #[inline(always)] - fn size_dynamic(&self) -> usize { - 0 - } - - #[inline(always)] - fn encode_static(&self, _buffer: &mut O) -> Result<(), Error> { - Ok(()) - } -} - -impl Deserialize for () { - fn decode_static(_buffer: &mut I) -> Result { - Ok(()) - } -} - -/// To protect against malicious large inputs, vector size is limited when decoding. -pub const VEC_DECODE_LIMIT: usize = 100 * (1 << 20); // 100 MiB - -#[cfg(feature = "alloc")] -impl Serialize for Vec { - fn size_static(&self) -> usize { - 8 - } - - #[inline(always)] - fn size_dynamic(&self) -> usize { - if T::UNALIGNED_BYTES { - aligned_size(self.len()) - } else { - aligned_size(self.iter().map(|e| e.size()).sum()) - } - } - - #[inline(always)] - // Encode only the size of the vector. Elements will be encoded in the - // `encode_dynamic` method. - fn encode_static(&self, buffer: &mut O) -> Result<(), Error> { - if self.len() > VEC_DECODE_LIMIT { - return Err(Error::AllocationLimit) - } - let len: u64 = self.len().try_into().expect("msg.len() > u64::MAX"); - len.encode(buffer) - } - - fn encode_dynamic(&self, buffer: &mut O) -> Result<(), Error> { - // Bytes - Vec it a separate case without padding for each element. - // It should padded at the end if is not % ALIGN - if T::UNALIGNED_BYTES { - // SAFETY: `UNALIGNED_BYTES` only set for `u8`. - let bytes = unsafe { ::core::mem::transmute::<&Vec, &Vec>(self) }; - buffer.write(bytes.as_slice())?; - for _ in 0..alignment_bytes(self.len()) { - buffer.push_byte(0)?; - } - } else { - for e in self.iter() { - e.encode(buffer)?; - } - } - Ok(()) - } -} - -#[cfg(feature = "alloc")] -impl Deserialize for Vec { - // Decode only the capacity of the vector. Elements will be decoded in the - // `decode_dynamic` method. The capacity is needed for iteration there. - fn decode_static(buffer: &mut I) -> Result { - let cap = u64::decode(buffer)?; - let cap: usize = cap.try_into().map_err(|_| Error::AllocationLimit)?; - if cap > VEC_DECODE_LIMIT { - return Err(Error::AllocationLimit) - } - Ok(Vec::with_capacity(cap)) - } - - fn decode_dynamic(&mut self, buffer: &mut I) -> Result<(), Error> { - for _ in 0..self.capacity() { - // Bytes - Vec it a separate case without unpadding for each element. - // It should unpadded at the end if is not % ALIGN - if T::UNALIGNED_BYTES { - let byte = buffer.read_byte()?; - // SAFETY: `UNALIGNED_BYTES` implemented set for `u8`. - let _self = - unsafe { ::core::mem::transmute::<&mut Vec, &mut Vec>(self) }; - _self.push(byte); - } else { - self.push(T::decode(buffer)?); - } - } - - if T::UNALIGNED_BYTES { - buffer.skip(alignment_bytes(self.capacity()))?; - } - - Ok(()) - } -} - -impl Serialize for [T; N] { - fn size_static(&self) -> usize { - self.iter().map(|e| e.size_static()).sum() - } - - #[inline(always)] - fn size_dynamic(&self) -> usize { - self.iter().map(|e| e.size_dynamic()).sum() - } - - #[inline(always)] - fn encode_static(&self, buffer: &mut O) -> Result<(), Error> { - let bytes = unsafe { ::core::mem::transmute::<&[T; N], &[u8; N]>(self) }; - buffer.write(bytes.as_slice())?; - Ok(()) - } - - fn encode_dynamic(&self, buffer: &mut O) -> Result<(), Error> { - for e in self.iter() { - e.encode_dynamic(buffer)?; - } - - Ok(()) - } -} - -impl DeserializeCompact for [T; N] { - fn decode_static(buffer: &mut I) -> Result { - let mut bytes: [u8; N] = [0; N]; - buffer.read(bytes.as_mut())?; - let ref_typed: &[T; N] = unsafe { core::mem::transmute(&bytes) }; - let typed: [T; N] = unsafe { core::ptr::read(ref_typed) }; - Ok(typed) - } - - fn decode_dynamic(&mut self, buffer: &mut I) -> Result<(), Error> { - for e in self.iter_mut() { - e.decode_dynamic(buffer)?; - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn validate(t: T) { - let bytes = t.to_bytes(); - let t2 = T::from_bytes(&bytes).expect("Roundtrip failed"); - assert_eq!(t, t2); - assert_eq!(t.to_bytes(), t2.to_bytes()); - - let mut vec = Vec::new(); - t.encode_static(&mut vec).expect("Encode failed"); - assert_eq!(vec.len(), t.size_static()); - } - - fn validate_enum(t: T) { - let bytes = t.to_bytes(); - let t2 = T::from_bytes(&bytes).expect("Roundtrip failed"); - assert_eq!(t, t2); - assert_eq!(t.to_bytes(), t2.to_bytes()); - - let mut vec = Vec::new(); - t.encode_static(&mut vec).expect("Encode failed"); - assert_eq!(vec.len(), t.size_static()); - t.encode_dynamic(&mut vec).expect("Encode failed"); - assert_eq!(vec.len(), t.size()); - - let mut vec2 = Vec::new(); - t.encode_dynamic(&mut vec2).expect("Encode failed"); - assert_eq!(vec2.len(), t.size_dynamic()); - } - - #[test] - fn test_compact_encode_decode() { - validate(()); - validate(123u8); - validate(u8::MAX); - validate(123u16); - validate(u16::MAX); - validate(123u32); - validate(u32::MAX); - validate(123u64); - validate(u64::MAX); - validate(123u128); - validate(u128::MAX); - validate(Vec::::new()); - validate(Vec::::new()); - validate(Vec::::new()); - validate(Vec::::new()); - validate(Vec::::new()); - validate(vec![1u8]); - validate(vec![1u16]); - validate(vec![1u32]); - validate(vec![1u64]); - validate(vec![1u128]); - validate(vec![1u8, 2u8]); - validate(vec![1u16, 2u16]); - validate(vec![1u32, 2u32]); - validate(vec![1u64, 2u64]); - validate(vec![1u128, 2u128]); - - #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] - struct TestStruct1 { - a: u8, - b: u16, - } - - let t = TestStruct1 { a: 123, b: 456 }; - assert_eq!(t.size_static(), 16); - assert_eq!(t.size(), 16); - validate(t); - - #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] - struct TestStruct2 { - a: u8, - v: Vec, - b: u16, - arr0: [u8; 0], - arr1: [u8; 2], - arr2: [u16; 3], - arr3: [u64; 4], - } - - validate(TestStruct2 { - a: 123, - v: vec![1, 2, 3], - b: 456, - arr0: [], - arr1: [1, 2], - arr2: [1, 2, u16::MAX], - arr3: [0, 3, 1111, u64::MAX], - }); - - #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] - #[repr(transparent)] - struct TestStruct3([u8; 64]); - - let t = TestStruct3([1; 64]); - assert_eq!(t.size_static(), 64); - assert_eq!(t.size(), 64); - validate(t); - - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] - #[canonical(prefix = 1u64)] - struct Prefixed1 { - a: [u8; 3], - b: Vec, - } - validate(Prefixed1 { - a: [1, 2, 3], - b: vec![4, 5, 6], - }); - - #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] - #[repr(u8)] - enum TestEnum1 { - A, - B, - C = 0x13, - D, - } - - validate(TestEnum1::A); - validate(TestEnum1::B); - validate(TestEnum1::C); - validate(TestEnum1::D); - - #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] - enum TestEnum2 { - A(u8), - B([u8; 3]), - C(Vec), - } - - validate_enum(TestEnum2::A(2)); - validate_enum(TestEnum2::B([1, 2, 3])); - validate_enum(TestEnum2::C(vec![1, 2, 3])); - - #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] - #[canonical(prefix = 2u64)] - struct Prefixed2(u16); - validate(Prefixed2(u16::MAX)); - - assert_eq!( - &Prefixed1 { - a: [1, 2, 3], - b: vec![4, 5] - } - .to_bytes()[..8], - &[0u8, 0, 0, 0, 0, 0, 0, 1] - ); - assert_eq!( - Prefixed2(u16::MAX).to_bytes(), - [0u8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0xff, 0xff] - ); - } -} diff --git a/fuel-types/src/numeric_types.rs b/fuel-types/src/numeric_types.rs index c27671b2e4..1ea18b3b04 100644 --- a/fuel-types/src/numeric_types.rs +++ b/fuel-types/src/numeric_types.rs @@ -32,7 +32,7 @@ macro_rules! key { #[repr(transparent)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(transparent))] - #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compact))] + #[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] #[cfg_attr(feature = "typescript", wasm_bindgen::prelude::wasm_bindgen)] #[derive( fuel_types::canonical::Serialize, fuel_types::canonical::Deserialize, diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index 052507d4c4..07155b004a 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -108,5 +108,4 @@ test-helpers = [ "dep:anyhow", "tai64", "fuel-crypto/test-helpers", - "fuel-compression?/test-helpers", ] From f0be2b120bde42bf5e73f539591e670857d5a15f Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 03:09:34 +0300 Subject: [PATCH 26/77] Complete migration to Green's trait-heavy approach --- fuel-compression/src/compaction.rs | 31 +++-- fuel-compression/src/lib.rs | 2 + fuel-derive/src/compact.rs | 108 +++++++++++------- fuel-tx/src/transaction/types/input.rs | 44 +++---- fuel-tx/src/transaction/types/input/coin.rs | 4 +- .../src/transaction/types/input/message.rs | 4 +- fuel-tx/src/transaction/types/mint.rs | 2 +- fuel-tx/src/transaction/types/output.rs | 10 +- fuel-tx/src/transaction/types/script.rs | 2 +- fuel-tx/src/transaction/types/witness.rs | 2 +- fuel-vm/Cargo.toml | 2 +- 11 files changed, 125 insertions(+), 86 deletions(-) diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 0e25ff831c..80e26bbefe 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -10,6 +10,8 @@ use serde::{ Serialize, }; +use crate::RawKey; + /// Convert data to reference-based format pub trait Compressible { /// The compacted type with references @@ -50,6 +52,21 @@ where fn decompress(&self, value: &Type::Compressed) -> anyhow::Result; } +pub trait RegistrySubstitutableBy: Compressible +where + Ctx: ?Sized, +{ + fn substitute(&self, ctx: &mut Ctx, keyspace: &str) -> anyhow::Result; +} + +pub trait RegistryDesubstitutableBy: Compressible +where + Ctx: ?Sized, + Self: Sized, +{ + fn desubstitute(c: &RawKey, ctx: &Ctx, keyspace: &str) -> anyhow::Result; +} + macro_rules! identity_compaction { ($t:ty) => { impl Compressible for $t { @@ -94,7 +111,7 @@ where T: CompressibleBy + Clone, { fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { - Ok(self.as_ref().map(|item| item.compress(ctx)).transpose()?) + self.as_ref().map(|item| item.compress(ctx)).transpose() } } @@ -179,7 +196,7 @@ where T: DecompressibleBy + Clone, { fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { - c.into_iter().map(|item| T::decompress(item, ctx)).collect() + c.iter().map(|item| T::decompress(item, ctx)).collect() } } @@ -187,19 +204,13 @@ impl Compressible for PhantomData { type Compressed = (); } -impl CompressibleBy for PhantomData -where - T: CompressibleBy + Clone, -{ +impl CompressibleBy for PhantomData { fn compress(&self, _: &mut Ctx) -> anyhow::Result { Ok(()) } } -impl DecompressibleBy for PhantomData -where - T: DecompressibleBy + Clone, -{ +impl DecompressibleBy for PhantomData { fn decompress(_: &Self::Compressed, _: &Ctx) -> anyhow::Result { Ok(PhantomData) } diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index 244b6e92d3..032a50fbdd 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -15,6 +15,8 @@ pub use compaction::{ CompressionContext, DecompressibleBy, DecompressionContext, + RegistryDesubstitutableBy, + RegistrySubstitutableBy, }; pub use key::RawKey; diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compact.rs index 1937b9e178..62522666b4 100644 --- a/fuel-derive/src/compact.rs +++ b/fuel-derive/src/compact.rs @@ -98,10 +98,15 @@ pub enum FieldAttrs { /// Compresseded recursively. Normal, /// This value is compacted into a registry lookup. - Registry, + Registry { + /// This is required to distinguish between values of the same type + keyspace: String, + }, } impl FieldAttrs { pub fn parse(attrs: &[syn::Attribute]) -> Self { + let registry_path = syn::parse2::(quote! {registry}).unwrap(); + let mut result = Self::Normal; for attr in attrs { if attr.style != syn::AttrStyle::Outer { @@ -118,9 +123,19 @@ impl FieldAttrs { if ident == "skip" { result = Self::Skip; continue; - } else if ident == "registry" { - result = Self::Registry; - continue; + } + } else if let Ok(kv) = + syn::parse2::(ml.tokens.clone()) + { + if kv.path == registry_path { + if let syn::Expr::Lit(lit) = kv.value { + if let syn::Lit::Str(keyspace) = lit.lit { + result = Self::Registry { + keyspace: keyspace.value(), + }; + continue; + } + } } } panic!("Invalid attribute: {}", ml.tokens); @@ -151,7 +166,7 @@ fn field_defs(fields: &syn::Fields) -> TokenStream2 { quote! { #cty, } } } - FieldAttrs::Registry => { + FieldAttrs::Registry { .. } => { if let Some(fname) = field.ident.as_ref() { quote! { #fname: ::fuel_compression::RawKey, } } else { @@ -189,9 +204,9 @@ fn construct_compact( let #cname = <#ty as ::fuel_compression::CompressibleBy<_>>::compress(&#binding, ctx)?; } } - FieldAttrs::Registry => { + FieldAttrs::Registry {keyspace} => { quote! { - let #cname = <#ty as ::fuel_compression::CompressibleBy<_>>::compress(&#binding, ctx)?; + let #cname = <#ty as ::fuel_compression::RegistrySubstitutableBy<_>>::substitute(&#binding, ctx, #keyspace)?; } } } @@ -249,9 +264,9 @@ fn construct_decompact( let #cname = <#ty as ::fuel_compression::DecompressibleBy<_>>::decompress(#binding, ctx)?; } } - FieldAttrs::Registry => { + FieldAttrs::Registry { keyspace } => { quote! { - let #cname = <#ty as ::fuel_compression::DecompressibleBy<_>>::decompress(#binding, ctx)?; + let #cname = <#ty as ::fuel_compression::RegistryDesubstitutableBy<_>>::desubstitute(#binding, ctx, #keyspace)?; } } } @@ -314,6 +329,19 @@ fn each_variant_compact) -> TokenStream2 .collect() } +fn where_clause_push(w: &mut Option, p: TokenStream2) { + if w.is_none() { + *w = Some(syn::WhereClause { + where_token: syn::Token![where](proc_macro2::Span::call_site()), + predicates: Default::default(), + }); + } + w.as_mut() + .unwrap() + .predicates + .push(syn::parse_quote! { #p }); +} + /// Derives `Compressed` trait for the given `struct` or `enum`. pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { s.add_bounds(synstructure::AddBounds::None) @@ -372,23 +400,21 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { for variant in s.variants() { for field in variant.ast().fields.iter() { let ty = &field.ty; - let attrs = FieldAttrs::parse(&field.attrs); - if matches!(attrs, FieldAttrs::Skip) { - continue; - } - - if w_impl_field_bounds_compress.is_none() { - w_impl_field_bounds_compress = Some(syn::WhereClause { - where_token: syn::Token![where](proc_macro2::Span::call_site()), - predicates: Default::default(), - }); + match FieldAttrs::parse(&field.attrs) { + FieldAttrs::Skip => {} + FieldAttrs::Normal => { + where_clause_push( + &mut w_impl_field_bounds_compress, + syn::parse_quote! { #ty: ::fuel_compression::CompressibleBy }, + ); + } + FieldAttrs::Registry { .. } => { + where_clause_push( + &mut w_impl_field_bounds_compress, + syn::parse_quote! { #ty: ::fuel_compression::RegistrySubstitutableBy }, + ); + } } - - w_impl_field_bounds_compress - .as_mut() - .unwrap() - .predicates - .push(syn::parse_quote! { #ty: ::fuel_compression::CompressibleBy }); } } @@ -396,25 +422,21 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { for variant in s.variants() { for field in variant.ast().fields.iter() { let ty = &field.ty; - let attrs = FieldAttrs::parse(&field.attrs); - if matches!(attrs, FieldAttrs::Skip) { - continue; - } - - if w_impl_field_bounds_decompress.is_none() { - w_impl_field_bounds_decompress = Some(syn::WhereClause { - where_token: syn::Token![where](proc_macro2::Span::call_site()), - predicates: Default::default(), - }); + match FieldAttrs::parse(&field.attrs) { + FieldAttrs::Skip => {} + FieldAttrs::Normal => { + where_clause_push( + &mut w_impl_field_bounds_decompress, + syn::parse_quote! { #ty: ::fuel_compression::DecompressibleBy }, + ); + } + FieldAttrs::Registry { .. } => { + where_clause_push( + &mut w_impl_field_bounds_decompress, + syn::parse_quote! { #ty: ::fuel_compression::RegistryDesubstitutableBy }, + ); + } } - - w_impl_field_bounds_decompress - .as_mut() - .unwrap() - .predicates - .push( - syn::parse_quote! { #ty: ::fuel_compression::DecompressibleBy }, - ); } } diff --git a/fuel-tx/src/transaction/types/input.rs b/fuel-tx/src/transaction/types/input.rs index 099ff5cb0f..4ceccdc09a 100644 --- a/fuel-tx/src/transaction/types/input.rs +++ b/fuel-tx/src/transaction/types/input.rs @@ -9,12 +9,10 @@ use alloc::{ use coin::*; use consts::*; use contract::*; -use core::{ - fmt, - fmt::Formatter, +use core::fmt::{ + self, + Formatter, }; -#[cfg(feature = "da-compression")] -use fuel_compression::Compressible; use fuel_crypto::{ Hasher, PublicKey, @@ -102,25 +100,31 @@ impl Deserialize for Empty { } #[cfg(feature = "da-compression")] -impl Compressible for Empty +impl fuel_compression::Compressible for Empty where - T: Compressible, + T: fuel_compression::Compressible, { type Compressed = (); +} + +#[cfg(feature = "da-compression")] +impl fuel_compression::CompressibleBy for Empty +where + T: fuel_compression::Compressible, +{ + fn compress(&self, _: &mut Ctx) -> anyhow::Result { + Ok(()) + } +} - // fn compact( - // &self, - // _: &mut dyn fuel_compression::CompressionContext, - // ) -> anyhow::Result { - // Ok(()) - // } - - // fn decompact( - // _: Self::Compressed, - // _: &dyn fuel_compression::DecompactionContext, - // ) -> anyhow::Result { - // Ok(Self(Default::default())) - // } +#[cfg(feature = "da-compression")] +impl fuel_compression::DecompressibleBy for Empty +where + T: fuel_compression::Compressible, +{ + fn decompress(_: &Self::Compressed, _: &Ctx) -> anyhow::Result { + Ok(Empty(::core::marker::PhantomData)) + } } impl AsFieldFmt for Empty { diff --git a/fuel-tx/src/transaction/types/input/coin.rs b/fuel-tx/src/transaction/types/input/coin.rs index 65a46a61ee..a6394786bf 100644 --- a/fuel-tx/src/transaction/types/input/coin.rs +++ b/fuel-tx/src/transaction/types/input/coin.rs @@ -114,10 +114,10 @@ where Specification: CoinSpecification + Clone, { pub utxo_id: UtxoId, - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "address"))] pub owner: Address, pub amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "asset_id"))] pub asset_id: AssetId, pub tx_pointer: TxPointer, #[derivative(Debug(format_with = "fmt_as_field"))] diff --git a/fuel-tx/src/transaction/types/input/message.rs b/fuel-tx/src/transaction/types/input/message.rs index cf549ff4da..3a2983e121 100644 --- a/fuel-tx/src/transaction/types/input/message.rs +++ b/fuel-tx/src/transaction/types/input/message.rs @@ -184,10 +184,10 @@ where Specification: MessageSpecification + Clone, { /// The sender from the L1 chain. - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "address"))] pub sender: Address, /// The receiver on the `Fuel` chain. - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "address"))] pub recipient: Address, pub amount: Word, pub nonce: Nonce, diff --git a/fuel-tx/src/transaction/types/mint.rs b/fuel-tx/src/transaction/types/mint.rs index 6dea93860e..1204426b64 100644 --- a/fuel-tx/src/transaction/types/mint.rs +++ b/fuel-tx/src/transaction/types/mint.rs @@ -64,7 +64,7 @@ pub struct Mint { /// The amount of funds minted. pub(crate) mint_amount: Word, /// The asset IDs corresponding to the minted amount. - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "asset_id"))] pub(crate) mint_asset_id: AssetId, /// Gas Price used for current block pub(crate) gas_price: Word, diff --git a/fuel-tx/src/transaction/types/output.rs b/fuel-tx/src/transaction/types/output.rs index 93b81f0dc2..ac039b2a88 100644 --- a/fuel-tx/src/transaction/types/output.rs +++ b/fuel-tx/src/transaction/types/output.rs @@ -25,21 +25,21 @@ pub use repr::OutputRepr; #[derive(canonical::Deserialize, canonical::Serialize)] pub enum Output { Coin { - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "address"))] to: Address, amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "asset_id"))] asset_id: AssetId, }, Contract(Contract), Change { - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "address"))] to: Address, #[cfg_attr(feature = "da-compression", da_compress(skip))] amount: Word, - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "asset_id"))] asset_id: AssetId, }, @@ -53,7 +53,7 @@ pub enum Output { }, ContractCreated { - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "contract_id"))] contract_id: ContractId, state_root: Bytes32, }, diff --git a/fuel-tx/src/transaction/types/script.rs b/fuel-tx/src/transaction/types/script.rs index d3af17d8c5..3e56dbd793 100644 --- a/fuel-tx/src/transaction/types/script.rs +++ b/fuel-tx/src/transaction/types/script.rs @@ -55,7 +55,7 @@ pub struct ScriptBody { pub(crate) script_gas_limit: Word, pub(crate) receipts_root: Bytes32, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "script_code"))] pub(crate) script: Vec, #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] pub(crate) script_data: Vec, diff --git a/fuel-tx/src/transaction/types/witness.rs b/fuel-tx/src/transaction/types/witness.rs index 7ce60edba5..7a2e5adba6 100644 --- a/fuel-tx/src/transaction/types/witness.rs +++ b/fuel-tx/src/transaction/types/witness.rs @@ -30,7 +30,7 @@ use rand::{ #[derive(fuel_types::canonical::Deserialize, fuel_types::canonical::Serialize)] pub struct Witness { #[derivative(Debug(format_with = "fmt_truncated_hex::<16>"))] - #[cfg_attr(feature = "da-compression", da_compress(registry))] + #[cfg_attr(feature = "da-compression", da_compress(registry = "witness"))] data: Vec, } diff --git a/fuel-vm/Cargo.toml b/fuel-vm/Cargo.toml index 07155b004a..f9a5d33c67 100644 --- a/fuel-vm/Cargo.toml +++ b/fuel-vm/Cargo.toml @@ -90,7 +90,7 @@ profile-gas = ["profile-any"] profile-coverage = ["profile-any"] profile-any = ["dyn-clone"] # All profiling features should depend on this random = ["fuel-crypto/random", "fuel-types/random", "fuel-tx/random", "rand"] -da-compression = ["fuel-compression", "fuel-tx/da-compression"] +da-compression = ["fuel-compression", "fuel-tx/da-compression", "fuel-types/da-compression"] serde = [ "dep:serde", "dep:serde_with", From 0b8a0d9455dc6da4066579b31952f5e5db493326 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 03:36:41 +0300 Subject: [PATCH 27/77] Polish: docs, naming, argument order --- fuel-compression/Cargo.toml | 6 - fuel-compression/src/compaction.rs | 351 ++---------------- fuel-compression/src/key.rs | 50 +-- fuel-derive/src/{compact.rs => compressed.rs} | 64 ++-- fuel-derive/src/lib.rs | 6 +- 5 files changed, 78 insertions(+), 399 deletions(-) rename fuel-derive/src/{compact.rs => compressed.rs} (90%) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index 00c02082e4..c0adbcc44d 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -15,9 +15,3 @@ anyhow = "1.0" fuel-derive = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" - -[dev-dependencies] -bimap = "0.6" -# postcard = { version = "1.0", features = ["use-std"] } -fuel-types = { workspace = true, features = ["serde"] } -# fuel-compression = { path = "." } \ No newline at end of file diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/compaction.rs index 80e26bbefe..b98bb11cb1 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/compaction.rs @@ -1,5 +1,3 @@ -#![allow(missing_docs)] // TODO: add documentation - use std::{ marker::PhantomData, mem::MaybeUninit, @@ -12,59 +10,84 @@ use serde::{ use crate::RawKey; -/// Convert data to reference-based format +/// This type can be compressed to a more compact form and back using +/// `CompressibleBy` and `DecompressibleBy` traits. pub trait Compressible { - /// The compacted type with references + /// The compressed type. type Compressed: Clone + Serialize + for<'a> Deserialize<'a>; } -/// This type is compressable by the given compression context +/// This type can be compressed to a more compact form and back using +/// `CompressionContext`. pub trait CompressibleBy: Compressible where Ctx: ?Sized, { - /// Perform compression, returning the compressed data, - /// modifying the context + /// Perform compression, returning the compressed data and possibly modifying the + /// context. The context is mutable to allow for stateful compression. + /// For instance, it can be used to extract original data when replacing it with + /// references. fn compress(&self, ctx: &mut Ctx) -> anyhow::Result; } -/// This type is decompressable by the given decompression context +/// This type can be decompressed using `CompressionContext`. pub trait DecompressibleBy: Compressible where Ctx: ?Sized, Self: Sized, { - /// Perform decompression, returning the original data + /// Perform decompression, returning the original data. + /// The context can be used to resolve references. fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result; } +/// A context that can be used to compress a type. pub trait CompressionContext where Type: Compressible, { + /// Perform compression, returning the compressed data and possibly modifying the + /// context. The context is mutable to allow for stateful compression. + /// For instance, it can be used to extract original data when replacing it with + /// references. fn compress(&mut self, value: &Type) -> anyhow::Result; } +/// A context that can be used to decompress a type. pub trait DecompressionContext where Type: Compressible, { + /// Perform decompression, returning the original data. + /// The context can be used to resolve references. fn decompress(&self, value: &Type::Compressed) -> anyhow::Result; } +/// Uses a compression context to substitute a type with a reference. +/// This is used instead of `CompressibleBy` when the type is substitutable by +/// a reference. Used with `da_compress(registry = "keyspace")` attribute from +/// `fuel-derive::Compressed`. pub trait RegistrySubstitutableBy: Compressible where Ctx: ?Sized, { - fn substitute(&self, ctx: &mut Ctx, keyspace: &str) -> anyhow::Result; + /// Perform substitution, returning the reference and possibly modifying the context. + /// Typically the original value is stored into the context. + fn substitute(&self, keyspace: &str, ctx: &mut Ctx) -> anyhow::Result; } +/// Uses a decompression context +/// This is used instead of `DecompressibleBy` when the type is desubstitutable from +/// a reference. Used with `da_compress(registry = "keyspace")` attribute from +/// `fuel-derive::Compressed`. pub trait RegistryDesubstitutableBy: Compressible where Ctx: ?Sized, Self: Sized, { - fn desubstitute(c: &RawKey, ctx: &Ctx, keyspace: &str) -> anyhow::Result; + /// Perform desubstitution, returning the original value. + /// The context is typically used to resolve the reference. + fn desubstitute(c: &RawKey, keyspace: &str, ctx: &Ctx) -> anyhow::Result; } macro_rules! identity_compaction { @@ -215,309 +238,3 @@ impl DecompressibleBy for PhantomData { Ok(PhantomData) } } - -#[cfg(feature = "never")] -// #[cfg(test)] -mod tests { - use std::collections::HashMap; - - use crate::RawKey; - - use super::{ - Compressible, - CompressibleBy, - CompressionContext, - DecompressibleBy, - DecompressionContext, - }; - use fuel_derive::Compressed; - use fuel_types::{ - Address, - AssetId, - }; - use serde::{ - Deserialize, - Serialize, - }; - - pub struct TestCompressionContext { - pub assets: HashMap, - } - - impl Compressible for AssetId { - type Compressed = RawKey; - } - - impl CompressibleBy for AssetId - where - Ctx: CompressionContext, - { - fn compress(&self, compressor: &mut Ctx) -> anyhow::Result { - compressor.compress(self) - } - } - - impl CompressionContext for TestCompressionContext { - fn compress( - &mut self, - type_to_compact: &AssetId, - ) -> anyhow::Result<::Compressed> { - let size = self.assets.len(); - let entry = self - .assets - .entry(*type_to_compact) - .or_insert_with(|| RawKey::try_from(size as u32).unwrap()); - Ok(*entry) - } - } - - #[derive(Debug, Clone, PartialEq)] - struct ManualExample { - a: AssetId, - b: AssetId, - c: u64, - d: [u8; 32], - } - - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] - struct ManualExampleCompressed { - a: ::Compressed, - b: ::Compressed, - c: ::Compressed, - c: <[u8; 32] as Compressible>::Compressed, - } - - impl Compressible for ManualExample { - type Compressed = ManualExampleCompressed; - } - - impl CompressibleBy for ManualExample { - fn compress(&self, ctx: &mut Ctx) -> anyhow::Result - where - AssetId: CompressibleBy, - u64: CompressibleBy, - [u8; 32]: CompressibleBy, - { - Ok(ManualExampleCompressed { - a: self.a.compress(ctx)?, - b: self.b.compress(ctx)?, - c: self.c.compress(ctx)?, - d: self.d.compress(ctx)?, - }) - } - } - - impl DecompressibleBy for ManualExample { - fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result - where - u64: DecompressibleBy, - { - Ok(ManualExample { - a: >::decompress(c.a, ctx)?, - b: >::decompress(c.b, ctx)?, - c: >::decompress(c.c, ctx)?, - d: <[u8; 32] as DedompressibleBy>::decompress(c.d, ctx)?, - }) - } - } - - // #[derive(Debug, Clone, PartialEq, Compressed)] - // struct AutomaticExample { - // #[da_compress(registry)] - // a: AssetId, - // #[da_compress(registry)] - // b: AssetId, - // c: u32, - // } - - #[test] - fn test_compaction_properties() { - let _a = ManualExample { - a: AssetId::from([1u8; 32]), - b: AssetId::from([2u8; 32]), - c: 3, - }; - - // let b = AutomaticExample { - // a: AssetId::from([1u8; 32]), - // b: AssetId::from([2u8; 32]), - // c: 3, - // }; - // assert_eq!(b.count().Address, 0); - // assert_eq!(b.count().AssetId, 2); - } - - // #[test] - // fn test_compaction_roundtrip_manual() { - // let target = ManualExample { - // a: Address::from([1u8; 32]), - // b: Address::from([2u8; 32]), - // c: 3, - // }; - // let mut registry = DummyRegistry::default(); - // let (compacted, _) = registry.compact(target.clone()).unwrap(); - // let decompacted = ManualExample::decompact(compacted, ®istry).unwrap(); - // assert_eq!(target, decompacted); - // } - - // #[test] - // fn test_compaction_roundtrip_derive() { - // let target = AutomaticExample { - // a: AssetId::from([1u8; 32]), - // b: AssetId::from([2u8; 32]), - // c: 3, - // }; - // let mut registry = DummyRegistry::default(); - // let (compacted, _) = registry.compact(target.clone()).unwrap(); - // let decompacted = AutomaticExample::decompact(compacted, ®istry).unwrap(); - // assert_eq!(target, decompacted); - // } -} - -#[cfg(test)] -mod playground { - use fuel_types::AssetId; - - use crate::RawKey; - - use super::{ - ArrayWrapper, - Compressible, - CompressibleBy, - CompressionContext, - DecompressibleBy, - DecompressionContext, - }; - - impl Compressible for AssetId { - type Compressed = RawKey; - } - - impl CompressibleBy for AssetId - where - C: CompressionContext, - { - fn compress(&self, compressor: &mut C) -> anyhow::Result { - compressor.compress(self) - } - } - - impl DecompressibleBy for AssetId - where - C: DecompressionContext, - { - fn decompress(c: &Self::Compressed, compressor: &C) -> anyhow::Result { - compressor.decompress(c) - } - } - - // #[derive(CompressibleBy)] - #[derive(Clone, Debug, PartialEq, Default)] - pub struct ComplexStruct { - asset_id: AssetId, - array: [u8; 32], - // #[compressible_by(skip)] - some_field: u64, - } - - // Generated code from `#[derive(CompressibleBy)]` - - #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] - pub struct CompressedComplexStruct { - asset_id: ::Compressed, - array: ArrayWrapper<32, u8>, - } - - impl Compressible for ComplexStruct { - type Compressed = CompressedComplexStruct; - } - - impl CompressibleBy for ComplexStruct - where - AssetId: CompressibleBy, - [u8; 32]: DecompressibleBy, - { - fn compress(&self, register: &mut C) -> anyhow::Result { - let asset_id = self.asset_id.compress(register)?; - let array = self.array.compress(register)?; - Ok(CompressedComplexStruct { asset_id, array }) - } - } - - impl DecompressibleBy for ComplexStruct - where - AssetId: DecompressibleBy, - [u8; 32]: DecompressibleBy, - { - fn decompress(c: &Self::Compressed, register: &C) -> anyhow::Result { - let asset_id = AssetId::decompress(&c.asset_id, register)?; - let array = - <[u8; 32] as DecompressibleBy>::decompress(&c.array, register)?; - Ok(ComplexStruct { - asset_id, - array, - ..Default::default() - }) - } - } - - mod tests { - use bimap::BiMap; - - use super::*; - - #[derive(Default)] - struct MapRegister { - assets: BiMap, - } - - impl CompressionContext for MapRegister { - fn compress(&mut self, type_to_compact: &AssetId) -> anyhow::Result { - if let Some(key) = self.assets.get_by_right(type_to_compact) { - return Ok(*key); - } - let size = self.assets.len(); - let key = RawKey::try_from(size as u32).unwrap(); - self.assets.insert(key, *type_to_compact); - Ok(key) - } - } - - impl DecompressionContext for MapRegister { - fn decompress(&self, c: &RawKey) -> anyhow::Result { - self.assets - .get_by_left(c) - .copied() - .ok_or_else(|| anyhow::anyhow!("Asset not found")) - } - } - - #[test] - fn can_register() { - let mut register = MapRegister::default(); - let complex_struct = ComplexStruct { - asset_id: [1; 32].into(), - array: [2; 32], - some_field: Default::default(), - }; - let compressed_complex = complex_struct.compress(&mut register).unwrap(); - let restored = >::decompress( - &compressed_complex, - ®ister, - ) - .unwrap(); - assert_eq!(complex_struct, restored); - - let compressed_again_complex = - complex_struct.compress(&mut register).unwrap(); - - let new_complex_struct = ComplexStruct { - asset_id: [2; 32].into(), - array: [2; 32], - some_field: 3, - }; - let compressed_new_complex = - new_complex_struct.compress(&mut register).unwrap(); - } - } -} diff --git a/fuel-compression/src/key.rs b/fuel-compression/src/key.rs index ab00d9db73..906b5d6cf6 100644 --- a/fuel-compression/src/key.rs +++ b/fuel-compression/src/key.rs @@ -24,49 +24,17 @@ impl RawKey { } /// Wraps around just below max/default value. - #[allow(clippy::cast_possible_truncation)] - pub fn add_u32(self, rhs: u32) -> Self { - let lhs = self.as_u32() as u64; - let rhs = rhs as u64; - // Safety: cannot overflow as both operands are limited to 32 bits - let result = (lhs + rhs) % (Self::DEFAULT_VALUE.as_u32() as u64); - // Safety: cannot truncate as we are already limited to 24 bits by modulo - let v = result as u32; - let v = v.to_be_bytes(); - Self([v[1], v[2], v[3]]) - } - - /// Wraps around just below max/default value. + /// Panics for max/default value. pub fn next(self) -> Self { - self.add_u32(1) - } - - /// Is `self` between `start` and `end`? i.e. in the half-open logical range - /// `start`..`end`, so that wrap-around cases are handled correctly. - /// - /// Panics if max/default value is used. - pub fn is_between(self, start: Self, end: Self) -> bool { - assert!( - self != Self::DEFAULT_VALUE, - "Cannot use max/default value in is_between" - ); - assert!( - start != Self::DEFAULT_VALUE, - "Cannot use max/default value in is_between" - ); - assert!( - end != Self::DEFAULT_VALUE, - "Cannot use max/default value in is_between" - ); - - let low = start.as_u32(); - let high = end.as_u32(); - let v = self.as_u32(); - - if high >= low { - low <= v && v < high + if self == Self::DEFAULT_VALUE { + panic!("Max/default value has no next key"); + } + let next_raw = self.as_u32() + 1u32; + if next_raw == Self::DEFAULT_VALUE.as_u32() { + Self::ZERO } else { - v < high || v >= low + Self::try_from(next_raw) + .expect("The producedure above always produces a valid key") } } } diff --git a/fuel-derive/src/compact.rs b/fuel-derive/src/compressed.rs similarity index 90% rename from fuel-derive/src/compact.rs rename to fuel-derive/src/compressed.rs index 62522666b4..a81ea746d4 100644 --- a/fuel-derive/src/compact.rs +++ b/fuel-derive/src/compressed.rs @@ -93,11 +93,11 @@ impl StructureAttrs { /// Field attributes pub enum FieldAttrs { - /// Skipped when compacting, and must be reconstructed when decompacting. + /// Skipped when compressing, and must be reconstructed when decompressing. Skip, /// Compresseded recursively. Normal, - /// This value is compacted into a registry lookup. + /// This value is compressed into a registry lookup. Registry { /// This is required to distinguish between values of the same type keyspace: String, @@ -147,7 +147,7 @@ impl FieldAttrs { } } -/// Map field definitions to compacted field definitions. +/// Map field definitions to compressed field definitions. fn field_defs(fields: &syn::Fields) -> TokenStream2 { let mut defs = TokenStream2::new(); @@ -183,10 +183,10 @@ fn field_defs(fields: &syn::Fields) -> TokenStream2 { } } -/// Construct compact version of the struct from the original one -fn construct_compact( +/// Construct compressed version of the struct from the original one +fn construct_compressed( // The structure to construct, i.e. struct name or enum variant path - compact: &TokenStream2, + compressed: &TokenStream2, variant: &synstructure::VariantInfo<'_>, ) -> TokenStream2 { let bound_fields: TokenStream2 = variant @@ -206,7 +206,7 @@ fn construct_compact( } FieldAttrs::Registry {keyspace} => { quote! { - let #cname = <#ty as ::fuel_compression::RegistrySubstitutableBy<_>>::substitute(&#binding, ctx, #keyspace)?; + let #cname = <#ty as ::fuel_compression::RegistrySubstitutableBy<_>>::substitute(&#binding, #keyspace, ctx)?; } } } @@ -238,11 +238,11 @@ fn construct_compact( quote! { #bound_fields - #compact #construct_fields + #compressed #construct_fields } } -/// Construct original version of the struct from the compacted one -fn construct_decompact( +/// Construct original version of the struct from the compressed one +fn construct_decompress( // The original structure to construct, i.e. struct name or enum variant path original: &TokenStream2, variant: &synstructure::VariantInfo<'_>, @@ -266,7 +266,7 @@ fn construct_decompact( } FieldAttrs::Registry { keyspace } => { quote! { - let #cname = <#ty as ::fuel_compression::RegistryDesubstitutableBy<_>>::desubstitute(#binding, ctx, #keyspace)?; + let #cname = <#ty as ::fuel_compression::RegistryDesubstitutableBy<_>>::desubstitute(#binding, #keyspace, ctx)?; } } } @@ -298,17 +298,17 @@ fn construct_decompact( } } -/// Generate a match arm for each variant of the compacted structure +/// Generate a match arm for each variant of the compressed structure /// using the given function to generate the pattern body. -fn each_variant_compact) -> TokenStream2>( +fn each_variant_compressed) -> TokenStream2>( s: &synstructure::Structure, - compact_name: &TokenStream2, + compressed_name: &TokenStream2, mut f: F, ) -> TokenStream2 { s.variants() .iter() .map(|variant| { - // Modify the binding pattern to match the compact variant + // Modify the binding pattern to match the compressed variant let mut v2 = variant.clone(); v2.filter(|field| { let attrs = FieldAttrs::parse(&field.ast().attrs); @@ -319,11 +319,11 @@ fn each_variant_compact) -> TokenStream2 }); let mut p = v2.pat().into_iter(); let _ = p.next().expect("pattern always begins with an identifier"); - let p = quote! { #compact_name #(#p)* }; + let p = quote! { #compressed_name #(#p)* }; - let decompacted = f(variant); + let decompressed = f(variant); quote! { - #p => { #decompacted } + #p => { #decompressed } } }) .collect() @@ -343,7 +343,7 @@ fn where_clause_push(w: &mut Option, p: TokenStream2) { } /// Derives `Compressed` trait for the given `struct` or `enum`. -pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { +pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { s.add_bounds(synstructure::AddBounds::None) .underscore_const(true); @@ -353,7 +353,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { }; let name = &s.ast().ident; - let compact_name = format_ident!("Compressed{}", name); + let compressed_name = format_ident!("Compressed{}", name); let mut g = s.ast().generics.clone(); let mut w_structure = g.where_clause.take(); @@ -452,7 +452,7 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { quote! { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[doc = concat!("Compresseded version of `", stringify!(#name), "`.")] - pub struct #compact_name #g #w_structure #defs #semi + pub struct #compressed_name #g #w_structure #defs #semi } } syn::Data::Enum(_) => { @@ -471,46 +471,46 @@ pub fn compact_derive(mut s: synstructure::Structure) -> TokenStream2 { quote! { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[doc = concat!("Compresseded version of `", stringify!(#name), "`.")] - pub enum #compact_name #g #w_structure { #variant_defs } + pub enum #compressed_name #g #w_structure { #variant_defs } } } syn::Data::Union(_) => panic!("unions are not supported"), }; - let construct_per_variant = s.each_variant(|variant| { + let compress_per_variant = s.each_variant(|variant| { let vname = variant.ast().ident.clone(); let construct = match &s.ast().data { - syn::Data::Struct(_) => quote! { #compact_name }, - syn::Data::Enum(_) => quote! {#compact_name :: #vname }, + syn::Data::Struct(_) => quote! { #compressed_name }, + syn::Data::Enum(_) => quote! {#compressed_name :: #vname }, syn::Data::Union(_) => unreachable!(), }; - construct_compact(&construct, variant) + construct_compressed(&construct, variant) }); - let decompact_per_variant = - each_variant_compact(&s, "e! {#compact_name}, |variant| { + let decompress_per_variant = + each_variant_compressed(&s, "e! {#compressed_name}, |variant| { let vname = variant.ast().ident.clone(); let construct = match &s.ast().data { syn::Data::Struct(_) => quote! { #name }, syn::Data::Enum(_) => quote! {#name :: #vname }, syn::Data::Union(_) => unreachable!(), }; - construct_decompact(&construct, variant) + construct_decompress(&construct, variant) }); let impls = s.gen_impl(quote! { gen impl ::fuel_compression::Compressible for @Self #w_impl { - type Compressed = #compact_name #g; + type Compressed = #compressed_name #g; } gen impl ::fuel_compression::CompressibleBy for @Self #w_impl_field_bounds_compress { fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { - Ok(match self { #construct_per_variant }) + Ok(match self { #compress_per_variant }) } } gen impl ::fuel_compression::DecompressibleBy for @Self #w_impl_field_bounds_decompress { fn decompress(compressed: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { - Ok(match compressed { #decompact_per_variant }) + Ok(match compressed { #decompress_per_variant }) } } }); diff --git a/fuel-derive/src/lib.rs b/fuel-derive/src/lib.rs index 7b261fa320..511fd23c10 100644 --- a/fuel-derive/src/lib.rs +++ b/fuel-derive/src/lib.rs @@ -11,12 +11,12 @@ extern crate proc_macro; mod canonical_attribute; -mod compact; +mod compressed; mod deserialize; mod serialize; use self::{ - compact::compact_derive, + compressed::compressed_derive, deserialize::deserialize_derive, serialize::serialize_derive, }; @@ -34,5 +34,5 @@ synstructure::decl_derive!( synstructure::decl_derive!( [Compressed, attributes(da_compress)] => /// Derives `Compressed` trait for the given `struct` or `enum`. - compact_derive + compressed_derive ); From 69ac157e872a51140011b5d0a008c7a9acdc6c68 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 03:41:12 +0300 Subject: [PATCH 28/77] Clean up proc macro a bit --- fuel-derive/src/compressed.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/fuel-derive/src/compressed.rs b/fuel-derive/src/compressed.rs index a81ea746d4..01c8b23e11 100644 --- a/fuel-derive/src/compressed.rs +++ b/fuel-derive/src/compressed.rs @@ -94,10 +94,12 @@ impl StructureAttrs { /// Field attributes pub enum FieldAttrs { /// Skipped when compressing, and must be reconstructed when decompressing. + /// `#[da_compress(skip)]` Skip, /// Compresseded recursively. Normal, /// This value is compressed into a registry lookup. + /// `#[da_compress(registry = "keyspace")]` Registry { /// This is required to distinguish between values of the same type keyspace: String, @@ -361,22 +363,14 @@ pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { for item in &s_attrs { match item { StructureAttrs::Bound(bound) => { - if w_structure.is_none() { - w_structure = Some(syn::WhereClause { - where_token: syn::Token![where](proc_macro2::Span::call_site()), - predicates: Default::default(), - }); - w_impl = Some(syn::WhereClause { - where_token: syn::Token![where](proc_macro2::Span::call_site()), - predicates: Default::default(), - }); - } for p in bound { let id = syn::Ident::new(p, Span::call_site()); - w_structure.as_mut().unwrap().predicates.push( + where_clause_push( + &mut w_structure, syn::parse_quote! { #id: ::fuel_compression::Compressible }, ); - w_impl.as_mut().unwrap().predicates.push( + where_clause_push( + &mut w_impl, syn::parse_quote! { for<'de> #id: ::fuel_compression::Compressible + serde::Serialize + serde::Deserialize<'de> + Clone }, ); } From 1ef68030df13fc68116fa185d66dcdf531da2e71 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 03:56:47 +0300 Subject: [PATCH 29/77] Line count reduction :\ --- fuel-compression/README.md | 34 ------- .../src/{compaction.rs => impls.rs} | 92 ++----------------- fuel-compression/src/lib.rs | 13 +-- fuel-compression/src/traits.rs | 86 +++++++++++++++++ fuel-derive/README.md | 2 +- 5 files changed, 96 insertions(+), 131 deletions(-) delete mode 100644 fuel-compression/README.md rename fuel-compression/src/{compaction.rs => impls.rs} (57%) create mode 100644 fuel-compression/src/traits.rs diff --git a/fuel-compression/README.md b/fuel-compression/README.md deleted file mode 100644 index 23767f0ea1..0000000000 --- a/fuel-compression/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Compression and decompression of fuel-types for the DA layer - -## Compressed block header - -Each compressed block begins with a single-byte version field, so that it's possible to change the format later. - -## Temporal registry - -This crate provides offchain registries for different types such as `AssetId`, `ContractId`, scripts, and predicates. Each registry is a key-value store with three-byte key. The registires are essentially compression caches. The three byte key allows cache size of 16 million values before reregistering the older values. - -The registries allow replacing repeated objects with their respective keys, so if an object -is used multiple times in a short interval (couple of months, maybe), then the full value -exists on only a single uncompressed block, - -### Fraud proofs - -Compressed block will start with 32 bytes of merkle root over all compression smts, followed by newly registered values along with their keys. Using an SMT provides flexibility around the algorithm we use to define keys without knowing how exactly values were chosen to be registered. - -Each registry also uses an SMT. Since the keys are three bytes long, the depth of the SMT is capped at 24 levels. - - - - More efficient for fraud proofs instead of needing to provide entire previous blocks with proofs - -## Compression of `UtxoIds` - -Since each `UtxoId` only appears once, there's no point in registering them. Instead, they are replaced with `TxPointer`s (7 bytes worst case), which are still unique. - -### Fraud proofs - -During fraud proofs we need to use the `prev_root` to prove that the referenced block height is part of the chain. - -## Other techniques - -- These techniques should be good enough for now, but there are lots of other interesting ideas for this. diff --git a/fuel-compression/src/compaction.rs b/fuel-compression/src/impls.rs similarity index 57% rename from fuel-compression/src/compaction.rs rename to fuel-compression/src/impls.rs index b98bb11cb1..787388f048 100644 --- a/fuel-compression/src/compaction.rs +++ b/fuel-compression/src/impls.rs @@ -1,94 +1,14 @@ -use std::{ - marker::PhantomData, - mem::MaybeUninit, -}; +//! Trait impls for Rust types +use crate::traits::*; use serde::{ Deserialize, Serialize, }; - -use crate::RawKey; - -/// This type can be compressed to a more compact form and back using -/// `CompressibleBy` and `DecompressibleBy` traits. -pub trait Compressible { - /// The compressed type. - type Compressed: Clone + Serialize + for<'a> Deserialize<'a>; -} - -/// This type can be compressed to a more compact form and back using -/// `CompressionContext`. -pub trait CompressibleBy: Compressible -where - Ctx: ?Sized, -{ - /// Perform compression, returning the compressed data and possibly modifying the - /// context. The context is mutable to allow for stateful compression. - /// For instance, it can be used to extract original data when replacing it with - /// references. - fn compress(&self, ctx: &mut Ctx) -> anyhow::Result; -} - -/// This type can be decompressed using `CompressionContext`. -pub trait DecompressibleBy: Compressible -where - Ctx: ?Sized, - Self: Sized, -{ - /// Perform decompression, returning the original data. - /// The context can be used to resolve references. - fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result; -} - -/// A context that can be used to compress a type. -pub trait CompressionContext -where - Type: Compressible, -{ - /// Perform compression, returning the compressed data and possibly modifying the - /// context. The context is mutable to allow for stateful compression. - /// For instance, it can be used to extract original data when replacing it with - /// references. - fn compress(&mut self, value: &Type) -> anyhow::Result; -} - -/// A context that can be used to decompress a type. -pub trait DecompressionContext -where - Type: Compressible, -{ - /// Perform decompression, returning the original data. - /// The context can be used to resolve references. - fn decompress(&self, value: &Type::Compressed) -> anyhow::Result; -} - -/// Uses a compression context to substitute a type with a reference. -/// This is used instead of `CompressibleBy` when the type is substitutable by -/// a reference. Used with `da_compress(registry = "keyspace")` attribute from -/// `fuel-derive::Compressed`. -pub trait RegistrySubstitutableBy: Compressible -where - Ctx: ?Sized, -{ - /// Perform substitution, returning the reference and possibly modifying the context. - /// Typically the original value is stored into the context. - fn substitute(&self, keyspace: &str, ctx: &mut Ctx) -> anyhow::Result; -} - -/// Uses a decompression context -/// This is used instead of `DecompressibleBy` when the type is desubstitutable from -/// a reference. Used with `da_compress(registry = "keyspace")` attribute from -/// `fuel-derive::Compressed`. -pub trait RegistryDesubstitutableBy: Compressible -where - Ctx: ?Sized, - Self: Sized, -{ - /// Perform desubstitution, returning the original value. - /// The context is typically used to resolve the reference. - fn desubstitute(c: &RawKey, keyspace: &str, ctx: &Ctx) -> anyhow::Result; -} +use std::{ + marker::PhantomData, + mem::MaybeUninit, +}; macro_rules! identity_compaction { ($t:ty) => { diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs index 032a50fbdd..4b21c11408 100644 --- a/fuel-compression/src/lib.rs +++ b/fuel-compression/src/lib.rs @@ -6,18 +6,11 @@ #![deny(unused_crate_dependencies)] #![deny(clippy::cast_possible_truncation)] -mod compaction; +mod impls; mod key; +mod traits; -pub use compaction::{ - Compressible, - CompressibleBy, - CompressionContext, - DecompressibleBy, - DecompressionContext, - RegistryDesubstitutableBy, - RegistrySubstitutableBy, -}; pub use key::RawKey; +pub use traits::*; pub use fuel_derive::Compressed; diff --git a/fuel-compression/src/traits.rs b/fuel-compression/src/traits.rs new file mode 100644 index 0000000000..88d97502f2 --- /dev/null +++ b/fuel-compression/src/traits.rs @@ -0,0 +1,86 @@ +use serde::{ + Deserialize, + Serialize, +}; + +use crate::RawKey; + +/// This type can be compressed to a more compact form and back using +/// `CompressibleBy` and `DecompressibleBy` traits. +pub trait Compressible { + /// The compressed type. + type Compressed: Clone + Serialize + for<'a> Deserialize<'a>; +} + +/// This type can be compressed to a more compact form and back using +/// `CompressionContext`. +pub trait CompressibleBy: Compressible +where + Ctx: ?Sized, +{ + /// Perform compression, returning the compressed data and possibly modifying the + /// context. The context is mutable to allow for stateful compression. + /// For instance, it can be used to extract original data when replacing it with + /// references. + fn compress(&self, ctx: &mut Ctx) -> anyhow::Result; +} + +/// This type can be decompressed using `CompressionContext`. +pub trait DecompressibleBy: Compressible +where + Ctx: ?Sized, + Self: Sized, +{ + /// Perform decompression, returning the original data. + /// The context can be used to resolve references. + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result; +} + +/// A context that can be used to compress a type. +pub trait CompressionContext +where + Type: Compressible, +{ + /// Perform compression, returning the compressed data and possibly modifying the + /// context. The context is mutable to allow for stateful compression. + /// For instance, it can be used to extract original data when replacing it with + /// references. + fn compress(&mut self, value: &Type) -> anyhow::Result; +} + +/// A context that can be used to decompress a type. +pub trait DecompressionContext +where + Type: Compressible, +{ + /// Perform decompression, returning the original data. + /// The context can be used to resolve references. + fn decompress(&self, value: &Type::Compressed) -> anyhow::Result; +} + +/// Uses a compression context to substitute a type with a reference. +/// This is used instead of `CompressibleBy` when the type is substitutable by +/// a reference. Used with `da_compress(registry = "keyspace")` attribute from +/// `fuel-derive::Compressed`. +pub trait RegistrySubstitutableBy: Compressible +where + Ctx: ?Sized, +{ + /// Perform substitution, returning the reference and possibly modifying the context. + /// Typically the original value is stored into the context. + fn substitute(&self, keyspace: &str, ctx: &mut Ctx) -> anyhow::Result; +} + +/// Uses a decompression context +/// This is used instead of `DecompressibleBy` when the type is desubstitutable from +/// a reference. Used with `da_compress(registry = "keyspace")` attribute from +/// `fuel-derive::Compressed`. +pub trait RegistryDesubstitutableBy: Compressible +where + Ctx: ?Sized, + Self: Sized, +{ + /// Perform desubstitution, returning the original value. + /// The context is typically used to resolve the reference. + fn desubstitute(c: &RawKey, keyspace: &str, ctx: &Ctx) -> anyhow::Result; +} diff --git a/fuel-derive/README.md b/fuel-derive/README.md index d28edae921..3fef87bc3d 100644 --- a/fuel-derive/README.md +++ b/fuel-derive/README.md @@ -5,4 +5,4 @@ [![docs](https://docs.rs/fuel-derive/badge.svg)](https://docs.rs/fuel-derive/) [![discord](https://img.shields.io/badge/chat%20on-discord-orange?&logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/xfpK4Pe) -This crate contains derive macros for canonical serialization and deserialization. This is used with [`fuel-types/src/canonical.rs`](fuel-types/src/canonical.rs) module which contains the associated traits and their implementations for native Rust types. +This crate contains derive macros for canonical serialization and deserialization. This is used with [`fuel-types/src/canonical.rs`](fuel-types/src/canonical.rs) module which contains the associated traits and their implementations for native Rust types. It also contains compaction macro `Compressed`, exporeted by `fuel-compression`. From 07391b233d84bd04450a50634520fa79cbbe5bff Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 04:05:54 +0300 Subject: [PATCH 30/77] More polish --- fuel-compression/README.md | 34 +++++++++++++++++++++++++ fuel-tx/src/transaction/types/input.rs | 35 +++++--------------------- 2 files changed, 40 insertions(+), 29 deletions(-) create mode 100644 fuel-compression/README.md diff --git a/fuel-compression/README.md b/fuel-compression/README.md new file mode 100644 index 0000000000..23767f0ea1 --- /dev/null +++ b/fuel-compression/README.md @@ -0,0 +1,34 @@ +# Compression and decompression of fuel-types for the DA layer + +## Compressed block header + +Each compressed block begins with a single-byte version field, so that it's possible to change the format later. + +## Temporal registry + +This crate provides offchain registries for different types such as `AssetId`, `ContractId`, scripts, and predicates. Each registry is a key-value store with three-byte key. The registires are essentially compression caches. The three byte key allows cache size of 16 million values before reregistering the older values. + +The registries allow replacing repeated objects with their respective keys, so if an object +is used multiple times in a short interval (couple of months, maybe), then the full value +exists on only a single uncompressed block, + +### Fraud proofs + +Compressed block will start with 32 bytes of merkle root over all compression smts, followed by newly registered values along with their keys. Using an SMT provides flexibility around the algorithm we use to define keys without knowing how exactly values were chosen to be registered. + +Each registry also uses an SMT. Since the keys are three bytes long, the depth of the SMT is capped at 24 levels. + + + - More efficient for fraud proofs instead of needing to provide entire previous blocks with proofs + +## Compression of `UtxoIds` + +Since each `UtxoId` only appears once, there's no point in registering them. Instead, they are replaced with `TxPointer`s (7 bytes worst case), which are still unique. + +### Fraud proofs + +During fraud proofs we need to use the `prev_root` to prove that the referenced block height is part of the chain. + +## Other techniques + +- These techniques should be good enough for now, but there are lots of other interesting ideas for this. diff --git a/fuel-tx/src/transaction/types/input.rs b/fuel-tx/src/transaction/types/input.rs index 4ceccdc09a..f2cffc9165 100644 --- a/fuel-tx/src/transaction/types/input.rs +++ b/fuel-tx/src/transaction/types/input.rs @@ -57,7 +57,12 @@ pub trait AsField: AsFieldFmt { /// The empty field used by sub-types of the specification. #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Empty(::core::marker::PhantomData); +#[cfg_attr(feature = "da-compression", derive(fuel_compression::Compressed))] +#[cfg_attr(feature = "da-compression", da_compress(discard(Type)))] +pub struct Empty( + #[cfg_attr(feature = "da-compression", da_compress(skip))] + ::core::marker::PhantomData, +); impl Empty { /// Creates `Self`. @@ -99,34 +104,6 @@ impl Deserialize for Empty { } } -#[cfg(feature = "da-compression")] -impl fuel_compression::Compressible for Empty -where - T: fuel_compression::Compressible, -{ - type Compressed = (); -} - -#[cfg(feature = "da-compression")] -impl fuel_compression::CompressibleBy for Empty -where - T: fuel_compression::Compressible, -{ - fn compress(&self, _: &mut Ctx) -> anyhow::Result { - Ok(()) - } -} - -#[cfg(feature = "da-compression")] -impl fuel_compression::DecompressibleBy for Empty -where - T: fuel_compression::Compressible, -{ - fn decompress(_: &Self::Compressed, _: &Ctx) -> anyhow::Result { - Ok(Empty(::core::marker::PhantomData)) - } -} - impl AsFieldFmt for Empty { fn fmt_as_field(&self, f: &mut Formatter) -> fmt::Result { f.write_str("Empty") From 6b359f083cd544bcebd857c4c893ff363e0de475 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 04:06:54 +0300 Subject: [PATCH 31/77] Move readme from fuel-compression to fuel-core-compression --- fuel-compression/README.md | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 fuel-compression/README.md diff --git a/fuel-compression/README.md b/fuel-compression/README.md deleted file mode 100644 index 23767f0ea1..0000000000 --- a/fuel-compression/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Compression and decompression of fuel-types for the DA layer - -## Compressed block header - -Each compressed block begins with a single-byte version field, so that it's possible to change the format later. - -## Temporal registry - -This crate provides offchain registries for different types such as `AssetId`, `ContractId`, scripts, and predicates. Each registry is a key-value store with three-byte key. The registires are essentially compression caches. The three byte key allows cache size of 16 million values before reregistering the older values. - -The registries allow replacing repeated objects with their respective keys, so if an object -is used multiple times in a short interval (couple of months, maybe), then the full value -exists on only a single uncompressed block, - -### Fraud proofs - -Compressed block will start with 32 bytes of merkle root over all compression smts, followed by newly registered values along with their keys. Using an SMT provides flexibility around the algorithm we use to define keys without knowing how exactly values were chosen to be registered. - -Each registry also uses an SMT. Since the keys are three bytes long, the depth of the SMT is capped at 24 levels. - - - - More efficient for fraud proofs instead of needing to provide entire previous blocks with proofs - -## Compression of `UtxoIds` - -Since each `UtxoId` only appears once, there's no point in registering them. Instead, they are replaced with `TxPointer`s (7 bytes worst case), which are still unique. - -### Fraud proofs - -During fraud proofs we need to use the `prev_root` to prove that the referenced block height is part of the chain. - -## Other techniques - -- These techniques should be good enough for now, but there are lots of other interesting ideas for this. From 3ee758974a4f581585fc6ff0b69f766ea84fbe09 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 04:11:59 +0300 Subject: [PATCH 32/77] Enable more lints for fuel-derive, remove unnecessary regex dependency --- fuel-derive/Cargo.toml | 1 - fuel-derive/src/lib.rs | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fuel-derive/Cargo.toml b/fuel-derive/Cargo.toml index 17643dadab..a719ea0635 100644 --- a/fuel-derive/Cargo.toml +++ b/fuel-derive/Cargo.toml @@ -18,4 +18,3 @@ quote = "1" syn = { version = "2", features = ["full"] } proc-macro2 = "1" synstructure = "0.13" -regex = "1" diff --git a/fuel-derive/src/lib.rs b/fuel-derive/src/lib.rs index 511fd23c10..b9199b2c5d 100644 --- a/fuel-derive/src/lib.rs +++ b/fuel-derive/src/lib.rs @@ -1,6 +1,7 @@ //! Derive macros for canonical type serialization and deserialization. -#![deny(unused_must_use, missing_docs)] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![deny(unused_must_use, unsafe_code, unused_crate_dependencies, missing_docs)] #![deny( clippy::arithmetic_side_effects, clippy::cast_sign_loss, From 15307550d7fa947537aafd084533ee2ce0ed76fb Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 04:16:04 +0300 Subject: [PATCH 33/77] Remove debug writing to /tmp, as we're hopefully done here --- fuel-derive/src/compressed.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fuel-derive/src/compressed.rs b/fuel-derive/src/compressed.rs index 01c8b23e11..353b20990b 100644 --- a/fuel-derive/src/compressed.rs +++ b/fuel-derive/src/compressed.rs @@ -19,8 +19,10 @@ const ATTR: &str = "da_compress"; #[derive(Debug)] pub enum StructureAttrs { /// Insert bounds for a generic type + /// `#[da_compress(bound(Type))]` Bound(Vec), /// Discard generic parameter + /// `#[da_compress(discard(Type))]` Discard(Vec), } impl Parse for StructureAttrs { @@ -508,12 +510,8 @@ pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { } } }); - let rs = quote! { + quote! { #def #impls - }; - - let _ = std::fs::write(format!("/tmp/derive/{}.rs", name), rs.to_string()); - - rs + } } From f1e7114949d95b5f00af80558bf14bfd6da2dd2b Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 04:50:51 +0300 Subject: [PATCH 34/77] Get rid of anyhow --- fuel-compression/Cargo.toml | 1 - fuel-compression/src/impls.rs | 60 +++++++++++++++++----------------- fuel-compression/src/traits.rs | 58 +++++++++++++++++++------------- fuel-derive/src/compressed.rs | 24 +++++++------- fuel-tx/Cargo.toml | 3 +- fuel-types/Cargo.toml | 3 +- 6 files changed, 80 insertions(+), 69 deletions(-) diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml index c0adbcc44d..3c38089585 100644 --- a/fuel-compression/Cargo.toml +++ b/fuel-compression/Cargo.toml @@ -11,7 +11,6 @@ repository = { workspace = true } description = "Compression and decompression of Fuel blocks for DA storage." [dependencies] -anyhow = "1.0" fuel-derive = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" diff --git a/fuel-compression/src/impls.rs b/fuel-compression/src/impls.rs index 787388f048..694e72e72a 100644 --- a/fuel-compression/src/impls.rs +++ b/fuel-compression/src/impls.rs @@ -1,14 +1,14 @@ //! Trait impls for Rust types use crate::traits::*; +use core::{ + marker::PhantomData, + mem::MaybeUninit, +}; use serde::{ Deserialize, Serialize, }; -use std::{ - marker::PhantomData, - mem::MaybeUninit, -}; macro_rules! identity_compaction { ($t:ty) => { @@ -16,20 +16,20 @@ macro_rules! identity_compaction { type Compressed = Self; } - impl CompressibleBy for $t + impl CompressibleBy for $t where Ctx: ?Sized, { - fn compress(&self, _: &mut Ctx) -> anyhow::Result { + fn compress(&self, _: &mut Ctx) -> Result { Ok(*self) } } - impl DecompressibleBy for $t + impl DecompressibleBy for $t where Ctx: ?Sized, { - fn decompress(c: &Self::Compressed, _: &Ctx) -> anyhow::Result { + fn decompress(c: &Self::Compressed, _: &Ctx) -> Result { Ok(*c) } } @@ -49,20 +49,20 @@ where type Compressed = Option; } -impl CompressibleBy for Option +impl CompressibleBy for Option where - T: CompressibleBy + Clone, + T: CompressibleBy + Clone, { - fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { + fn compress(&self, ctx: &mut Ctx) -> Result { self.as_ref().map(|item| item.compress(ctx)).transpose() } } -impl DecompressibleBy for Option +impl DecompressibleBy for Option where - T: DecompressibleBy + Clone, + T: DecompressibleBy + Clone, { - fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> Result { c.as_ref().map(|item| T::decompress(item, ctx)).transpose() } } @@ -98,22 +98,22 @@ where type Compressed = ArrayWrapper; } -impl CompressibleBy for [T; S] +impl CompressibleBy for [T; S] where - T: CompressibleBy + Clone, + T: CompressibleBy + Clone, { - fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { + fn compress(&self, ctx: &mut Ctx) -> Result { Ok(ArrayWrapper(try_map_array(self.clone(), |v: T| { v.compress(ctx) })?)) } } -impl DecompressibleBy for [T; S] +impl DecompressibleBy for [T; S] where - T: DecompressibleBy + Clone, + T: DecompressibleBy + Clone, { - fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> Result { try_map_array(c.0.clone(), |v: T::Compressed| T::decompress(&v, ctx)) } } @@ -125,20 +125,20 @@ where type Compressed = Vec; } -impl CompressibleBy for Vec +impl CompressibleBy for Vec where - T: CompressibleBy + Clone, + T: CompressibleBy + Clone, { - fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { + fn compress(&self, ctx: &mut Ctx) -> Result { self.iter().map(|item| item.compress(ctx)).collect() } } -impl DecompressibleBy for Vec +impl DecompressibleBy for Vec where - T: DecompressibleBy + Clone, + T: DecompressibleBy + Clone, { - fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> Result { c.iter().map(|item| T::decompress(item, ctx)).collect() } } @@ -147,14 +147,14 @@ impl Compressible for PhantomData { type Compressed = (); } -impl CompressibleBy for PhantomData { - fn compress(&self, _: &mut Ctx) -> anyhow::Result { +impl CompressibleBy for PhantomData { + fn compress(&self, _: &mut Ctx) -> Result { Ok(()) } } -impl DecompressibleBy for PhantomData { - fn decompress(_: &Self::Compressed, _: &Ctx) -> anyhow::Result { +impl DecompressibleBy for PhantomData { + fn decompress(_: &Self::Compressed, _: &Ctx) -> Result { Ok(PhantomData) } } diff --git a/fuel-compression/src/traits.rs b/fuel-compression/src/traits.rs index 88d97502f2..ad080c5687 100644 --- a/fuel-compression/src/traits.rs +++ b/fuel-compression/src/traits.rs @@ -12,75 +12,89 @@ pub trait Compressible { type Compressed: Clone + Serialize + for<'a> Deserialize<'a>; } -/// This type can be compressed to a more compact form and back using -/// `CompressionContext`. -pub trait CompressibleBy: Compressible +/// A context that can be used to compress a type. +pub trait CompressionContext where - Ctx: ?Sized, + Type: Compressible, { + /// Error when compressing. Note that the compression itself is not faillible, + /// but the context may still do fallible operations. + type Error; + /// Perform compression, returning the compressed data and possibly modifying the /// context. The context is mutable to allow for stateful compression. /// For instance, it can be used to extract original data when replacing it with /// references. - fn compress(&self, ctx: &mut Ctx) -> anyhow::Result; + fn compress(&mut self, value: &Type) -> Result; } -/// This type can be decompressed using `CompressionContext`. -pub trait DecompressibleBy: Compressible +/// A context that can be used to decompress a type. +pub trait DecompressionContext where - Ctx: ?Sized, - Self: Sized, + Type: Compressible, { + /// Error when compressing. Note that the compression itself is not faillible, + /// but the context may still do fallible operations. + type Error; + /// Perform decompression, returning the original data. /// The context can be used to resolve references. - fn decompress(c: &Self::Compressed, ctx: &Ctx) -> anyhow::Result; + fn decompress(&self, value: &Type::Compressed) -> Result; } -/// A context that can be used to compress a type. -pub trait CompressionContext +/// Error type for context errors. +pub trait CtxError { + /// Context error type + type Error; +} + +/// This type can be compressed to a more compact form and back using +/// `CompressionContext`. +pub trait CompressibleBy: Compressible where - Type: Compressible, + Ctx: ?Sized, { /// Perform compression, returning the compressed data and possibly modifying the /// context. The context is mutable to allow for stateful compression. /// For instance, it can be used to extract original data when replacing it with /// references. - fn compress(&mut self, value: &Type) -> anyhow::Result; + fn compress(&self, ctx: &mut Ctx) -> Result; } -/// A context that can be used to decompress a type. -pub trait DecompressionContext +/// This type can be decompressed using `CompressionContext`. +pub trait DecompressibleBy: Compressible where - Type: Compressible, + Ctx: ?Sized, + Self: Sized, { /// Perform decompression, returning the original data. /// The context can be used to resolve references. - fn decompress(&self, value: &Type::Compressed) -> anyhow::Result; + fn decompress(c: &Self::Compressed, ctx: &Ctx) -> Result; } /// Uses a compression context to substitute a type with a reference. /// This is used instead of `CompressibleBy` when the type is substitutable by /// a reference. Used with `da_compress(registry = "keyspace")` attribute from /// `fuel-derive::Compressed`. -pub trait RegistrySubstitutableBy: Compressible +pub trait RegistrySubstitutableBy: Compressible where Ctx: ?Sized, { /// Perform substitution, returning the reference and possibly modifying the context. /// Typically the original value is stored into the context. - fn substitute(&self, keyspace: &str, ctx: &mut Ctx) -> anyhow::Result; + fn substitute(&self, keyspace: &str, ctx: &mut Ctx) -> Result; } /// Uses a decompression context /// This is used instead of `DecompressibleBy` when the type is desubstitutable from /// a reference. Used with `da_compress(registry = "keyspace")` attribute from /// `fuel-derive::Compressed`. -pub trait RegistryDesubstitutableBy: Compressible +pub trait RegistryDesubstitutableBy: Compressible where Ctx: ?Sized, Self: Sized, { /// Perform desubstitution, returning the original value. /// The context is typically used to resolve the reference. - fn desubstitute(c: &RawKey, keyspace: &str, ctx: &Ctx) -> anyhow::Result; + fn desubstitute(c: &RawKey, keyspace: &str, ctx: &Ctx) -> Result; } diff --git a/fuel-derive/src/compressed.rs b/fuel-derive/src/compressed.rs index 353b20990b..6c0629f84e 100644 --- a/fuel-derive/src/compressed.rs +++ b/fuel-derive/src/compressed.rs @@ -205,12 +205,12 @@ fn construct_compressed( FieldAttrs::Skip => quote! {}, FieldAttrs::Normal => { quote! { - let #cname = <#ty as ::fuel_compression::CompressibleBy<_>>::compress(&#binding, ctx)?; + let #cname = <#ty as ::fuel_compression::CompressibleBy<_, _>>::compress(&#binding, ctx)?; } } FieldAttrs::Registry {keyspace} => { quote! { - let #cname = <#ty as ::fuel_compression::RegistrySubstitutableBy<_>>::substitute(&#binding, #keyspace, ctx)?; + let #cname = <#ty as ::fuel_compression::RegistrySubstitutableBy<_, _>>::substitute(&#binding, #keyspace, ctx)?; } } } @@ -265,12 +265,12 @@ fn construct_decompress( }, FieldAttrs::Normal => { quote! { - let #cname = <#ty as ::fuel_compression::DecompressibleBy<_>>::decompress(#binding, ctx)?; + let #cname = <#ty as ::fuel_compression::DecompressibleBy<_, _>>::decompress(#binding, ctx)?; } } FieldAttrs::Registry { keyspace } => { quote! { - let #cname = <#ty as ::fuel_compression::RegistryDesubstitutableBy<_>>::desubstitute(#binding, #keyspace, ctx)?; + let #cname = <#ty as ::fuel_compression::RegistryDesubstitutableBy<_, _>>::desubstitute(#binding, #keyspace, ctx)?; } } } @@ -401,13 +401,13 @@ pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { FieldAttrs::Normal => { where_clause_push( &mut w_impl_field_bounds_compress, - syn::parse_quote! { #ty: ::fuel_compression::CompressibleBy }, + syn::parse_quote! { #ty: ::fuel_compression::CompressibleBy }, ); } FieldAttrs::Registry { .. } => { where_clause_push( &mut w_impl_field_bounds_compress, - syn::parse_quote! { #ty: ::fuel_compression::RegistrySubstitutableBy }, + syn::parse_quote! { #ty: ::fuel_compression::RegistrySubstitutableBy }, ); } } @@ -423,13 +423,13 @@ pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { FieldAttrs::Normal => { where_clause_push( &mut w_impl_field_bounds_decompress, - syn::parse_quote! { #ty: ::fuel_compression::DecompressibleBy }, + syn::parse_quote! { #ty: ::fuel_compression::DecompressibleBy }, ); } FieldAttrs::Registry { .. } => { where_clause_push( &mut w_impl_field_bounds_decompress, - syn::parse_quote! { #ty: ::fuel_compression::RegistryDesubstitutableBy }, + syn::parse_quote! { #ty: ::fuel_compression::RegistryDesubstitutableBy }, ); } } @@ -499,13 +499,13 @@ pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { type Compressed = #compressed_name #g; } - gen impl ::fuel_compression::CompressibleBy for @Self #w_impl_field_bounds_compress { - fn compress(&self, ctx: &mut Ctx) -> anyhow::Result { + gen impl ::fuel_compression::CompressibleBy for @Self #w_impl_field_bounds_compress { + fn compress(&self, ctx: &mut Ctx) -> Result { Ok(match self { #compress_per_variant }) } } - gen impl ::fuel_compression::DecompressibleBy for @Self #w_impl_field_bounds_decompress { - fn decompress(compressed: &Self::Compressed, ctx: &Ctx) -> anyhow::Result { + gen impl ::fuel_compression::DecompressibleBy for @Self #w_impl_field_bounds_decompress { + fn decompress(compressed: &Self::Compressed, ctx: &Ctx) -> Result { Ok(match compressed { #decompress_per_variant }) } } diff --git a/fuel-tx/Cargo.toml b/fuel-tx/Cargo.toml index d74f58bb6d..258bcd8b21 100644 --- a/fuel-tx/Cargo.toml +++ b/fuel-tx/Cargo.toml @@ -11,7 +11,6 @@ repository = { workspace = true } description = "FuelVM transaction." [dependencies] -anyhow = { version = "1.0", optional = true } bitflags = { workspace = true } derivative = { version = "2.2.0", default-features = false, features = ["use_core"], optional = true } derive_more = { version = "0.99", default-features = false, features = ["display"] } @@ -55,4 +54,4 @@ std = ["alloc", "fuel-asm/std", "fuel-crypto/std", "fuel-merkle/std", "fuel-type alloc = ["hashbrown", "fuel-types/alloc", "itertools/use_alloc", "derivative", "fuel-merkle", "strum", "strum_macros"] # serde is requiring alloc because its mandatory for serde_json. to avoid adding a new feature only for serde_json, we just require `alloc` here since as of the moment we don't have a use case of serde without alloc. serde = ["alloc", "fuel-asm/serde", "fuel-crypto/serde", "fuel-merkle/serde", "serde_json", "hashbrown/serde", "bitflags/serde"] -da-compression = ["anyhow", "serde", "fuel-compression", "fuel-types/da-compression"] +da-compression = ["serde", "fuel-compression", "fuel-types/da-compression"] diff --git a/fuel-types/Cargo.toml b/fuel-types/Cargo.toml index 3489390a67..9e053d3b19 100644 --- a/fuel-types/Cargo.toml +++ b/fuel-types/Cargo.toml @@ -11,7 +11,6 @@ repository = { workspace = true } description = "Atomic types of the FuelVM." [dependencies] -anyhow = { version = "1.0", optional = true } fuel-compression = { workspace = true, optional = true } fuel-derive = { workspace = true } hex = { version = "0.4", default-features = false } @@ -33,7 +32,7 @@ typescript = ["wasm-bindgen"] alloc = ["hex/alloc"] random = ["rand"] serde = ["dep:serde", "alloc"] -da-compression = ["anyhow", "serde", "fuel-compression"] +da-compression = ["serde", "fuel-compression"] std = ["alloc", "serde?/std", "hex/std"] unsafe = [] From 31083c1a78687a6c9047e26dfefb35ceb08ad7e9 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 06:21:54 +0300 Subject: [PATCH 35/77] Add roundtrip tests --- fuel-compression/src/traits.rs | 10 ++ fuel-tx/Cargo.toml | 1 + fuel-tx/src/tests/da_compression.rs | 214 ++++++++++++++++++++++++++++ fuel-tx/src/tests/mod.rs | 2 + 4 files changed, 227 insertions(+) create mode 100644 fuel-tx/src/tests/da_compression.rs diff --git a/fuel-compression/src/traits.rs b/fuel-compression/src/traits.rs index ad080c5687..955623a4a5 100644 --- a/fuel-compression/src/traits.rs +++ b/fuel-compression/src/traits.rs @@ -76,6 +76,11 @@ where /// This is used instead of `CompressibleBy` when the type is substitutable by /// a reference. Used with `da_compress(registry = "keyspace")` attribute from /// `fuel-derive::Compressed`. +#[diagnostic::on_unimplemented( + message = "fuel_compression::RegistrySubstitutableBy is not implemented for `{Self}`", + label = "When trying to compress this parent type", + note = "#[da_compress(registry = \"...\")] was likely used on field with type {Self}" +)] pub trait RegistrySubstitutableBy: Compressible where Ctx: ?Sized, @@ -89,6 +94,11 @@ where /// This is used instead of `DecompressibleBy` when the type is desubstitutable from /// a reference. Used with `da_compress(registry = "keyspace")` attribute from /// `fuel-derive::Compressed`. +#[diagnostic::on_unimplemented( + message = "fuel_compression::RegistryDesubstitutableBy is not implemented for `{Self}`", + label = "When trying to decompress this parent type", + note = "#[da_compress(registry = \"...\")] was likely used on field with type {Self}" +)] pub trait RegistryDesubstitutableBy: Compressible where Ctx: ?Sized, diff --git a/fuel-tx/Cargo.toml b/fuel-tx/Cargo.toml index 258bcd8b21..02f916b0d8 100644 --- a/fuel-tx/Cargo.toml +++ b/fuel-tx/Cargo.toml @@ -38,6 +38,7 @@ fuel-tx = { path = ".", features = ["random", "serde", "test-helpers"] } fuel-types = { workspace = true, default-features = false, features = ["random"] } hex = { version = "0.4", default-features = false } insta = "1.0" +postcard = { version = "1.0", features = ["use-std"] } quickcheck = "1.0" quickcheck_macros = "1.0" rand = { version = "0.8", default-features = false, features = ["std_rng"] } diff --git a/fuel-tx/src/tests/da_compression.rs b/fuel-tx/src/tests/da_compression.rs new file mode 100644 index 0000000000..53dd98c3f6 --- /dev/null +++ b/fuel-tx/src/tests/da_compression.rs @@ -0,0 +1,214 @@ +use crate::{ + builder::Finalizable, + test_helper::generate_bytes, + BlobBody, + BlobId, + ConsensusParameters, + Input, + Output, + Transaction, + TransactionBuilder, + UpgradePurpose, + UploadBody, +}; +use fuel_compression::{ + Compressed, + CompressibleBy, + DecompressibleBy, + RawKey, + RegistryDesubstitutableBy, + RegistrySubstitutableBy, +}; +use fuel_crypto::SecretKey; +use fuel_types::{ + Address, + AssetId, + ContractId, +}; +use rand::{ + rngs::StdRng, + Rng, + SeedableRng, +}; +use std::{ + collections::HashMap, + convert::Infallible, +}; + +/// A simple and inefficient registry for testing purposes +#[derive(Default)] +struct TestCompressionCtx { + registry: HashMap<(String, RawKey), Vec>, +} + +macro_rules! impl_substitutable { + ($t:ty) => { + impl RegistrySubstitutableBy for $t { + fn substitute( + &self, + keyspace: &str, + ctx: &mut TestCompressionCtx, + ) -> Result { + for ((ks, rk), v) in ctx.registry.iter() { + if ks != keyspace { + continue; + } + let d: $t = postcard::from_bytes(v).expect("failed to deserialize"); + if d == *self { + return Ok(*rk); + } + } + + let key = ctx.registry.len(); // Just get an unique integer key + let key = RawKey::try_from(key as u32).expect("key too large"); + let value = postcard::to_stdvec(self).expect("failed to serialize"); + ctx.registry.insert((keyspace.to_owned(), key), value); + Ok(key) + } + } + + impl RegistryDesubstitutableBy for $t { + fn desubstitute( + key: &RawKey, + keyspace: &str, + ctx: &TestCompressionCtx, + ) -> Result<$t, Infallible> { + let value = ctx + .registry + .get(&(keyspace.to_owned(), *key)) + .expect("key not found"); + Ok(postcard::from_bytes(value).expect("failed to deserialize")) + } + } + }; +} + +impl_substitutable!(Address); +impl_substitutable!(AssetId); +impl_substitutable!(ContractId); +impl_substitutable!(Vec); + +#[derive(Debug, PartialEq, Default, Compressed)] +pub struct ExampleStruct { + pub asset_id_bare: AssetId, + #[da_compress(registry = "assets")] + pub asset_id_ref: AssetId, + pub array: [u8; 32], + pub vec: Vec, + pub integer: u32, +} + +#[derive(Debug, PartialEq, Compressed)] +pub struct InnerStruct { + #[da_compress(registry = "assets")] + pub asset_id: AssetId, + pub count: u64, + #[da_compress(skip)] + pub cached: [u8; 32], +} + +#[test] +fn example_struct_roundtrip_simple() { + let mut ctx = TestCompressionCtx::default(); + let original = ExampleStruct::default(); + let compressed = original.compress(&mut ctx).expect("compression failed"); + let decompressed = + ExampleStruct::decompress(&compressed, &ctx).expect("decompression failed"); + assert_eq!(original, decompressed); +} + +#[test] +fn example_struct_postcard_roundtrip_multiple() { + let rng = &mut StdRng::seed_from_u64(8586); + + let mut ctx = TestCompressionCtx::default(); + for _ in 0..10 { + let original = ExampleStruct { + asset_id_bare: AssetId::new(rng.gen()), + asset_id_ref: AssetId::new(rng.gen()), + array: rng.gen(), + vec: (0..rng.gen_range(0..32)).map(|_| rng.gen::()).collect(), + integer: rng.gen(), + }; + let compressed = original.compress(&mut ctx).expect("compression failed"); + let postcard_compressed = + postcard::to_stdvec(&compressed).expect("failed to serialize"); + let postcard_decompressed = + postcard::from_bytes(&postcard_compressed).expect("failed to deserialize"); + let decompressed = ExampleStruct::decompress(&postcard_decompressed, &ctx) + .expect("decompression failed"); + assert_eq!(original, decompressed); + } +} + +#[test] +fn transaction_postcard_roundtrip() { + let rng = &mut StdRng::seed_from_u64(8586); + + // Malleable fields zero, others randomized. + let txs: Vec = vec![ + TransactionBuilder::script(generate_bytes(rng), generate_bytes(rng)) + .maturity(100u32.into()) + .add_random_fee_input() + .finalize() + .into(), + TransactionBuilder::create(generate_bytes(rng).into(), rng.gen(), vec![]) + .maturity(100u32.into()) + .add_unsigned_coin_input( + SecretKey::random(rng), + rng.gen(), + 0, + rng.gen(), + rng.gen(), + ) + .add_contract_created() + .add_output(Output::change(rng.gen(), 0, AssetId::default())) + .finalize() + .into(), + TransactionBuilder::upload(UploadBody { + root: rng.gen(), + witness_index: 0, + subsection_index: rng.gen(), + subsections_number: rng.gen(), + proof_set: Default::default(), + }) + .add_random_fee_input() + .finalize() + .into(), + TransactionBuilder::upgrade(UpgradePurpose::StateTransition { + root: Default::default(), + }) + .add_input(Input::coin_signed( + Default::default(), + *ConsensusParameters::standard().privileged_address(), + rng.gen(), + AssetId::BASE, + Default::default(), + 0, + )) + .add_random_fee_input() + .finalize() + .into(), + TransactionBuilder::blob(BlobBody { + id: BlobId::new(rng.gen()), + witness_index: 0, + }) + .add_witness(generate_bytes(rng).into()) + .maturity(Default::default()) + .add_random_fee_input() + .finalize() + .into(), + ]; + + let mut ctx = TestCompressionCtx::default(); + for tx in txs { + let compressed = tx.compress(&mut ctx).expect("compression failed"); + let postcard_compressed = + postcard::to_stdvec(&compressed).expect("failed to serialize"); + let postcard_decompressed = + postcard::from_bytes(&postcard_compressed).expect("failed to deserialize"); + let decompressed = Transaction::decompress(&postcard_decompressed, &ctx) + .expect("decompression failed"); + assert_eq!(tx, decompressed); + } +} diff --git a/fuel-tx/src/tests/mod.rs b/fuel-tx/src/tests/mod.rs index 5929a30612..3449de9d0d 100644 --- a/fuel-tx/src/tests/mod.rs +++ b/fuel-tx/src/tests/mod.rs @@ -5,6 +5,8 @@ mod valid_cases; #[cfg(feature = "serde")] mod bytes; +#[cfg(feature = "da-compression")] +mod da_compression; #[cfg(feature = "serde")] mod display; From 0cc0be078b5cd7c10d6fe84b62f2a98446d84ae0 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Tue, 3 Sep 2024 15:25:52 +0300 Subject: [PATCH 36/77] Use type-based registry keyspaces --- fuel-compression/src/traits.rs | 18 ++++---- fuel-derive/src/compressed.rs | 39 ++++++----------- fuel-tx/src/builder.rs | 2 +- fuel-tx/src/lib.rs | 1 + fuel-tx/src/tests/da_compression.rs | 22 +++++----- fuel-tx/src/transaction.rs | 2 +- fuel-tx/src/transaction/types.rs | 1 + fuel-tx/src/transaction/types/input/coin.rs | 4 +- .../src/transaction/types/input/message.rs | 4 +- fuel-tx/src/transaction/types/mint.rs | 2 +- fuel-tx/src/transaction/types/output.rs | 10 ++--- fuel-tx/src/transaction/types/script.rs | 42 ++++++++++++++++--- fuel-tx/src/transaction/types/witness.rs | 1 - 13 files changed, 82 insertions(+), 66 deletions(-) diff --git a/fuel-compression/src/traits.rs b/fuel-compression/src/traits.rs index 955623a4a5..fc34edad2c 100644 --- a/fuel-compression/src/traits.rs +++ b/fuel-compression/src/traits.rs @@ -74,12 +74,12 @@ where /// Uses a compression context to substitute a type with a reference. /// This is used instead of `CompressibleBy` when the type is substitutable by -/// a reference. Used with `da_compress(registry = "keyspace")` attribute from +/// a reference. Used with `da_compress(registry)` attribute from /// `fuel-derive::Compressed`. #[diagnostic::on_unimplemented( - message = "fuel_compression::RegistrySubstitutableBy is not implemented for `{Self}`", + message = "`fuel_compression::RegistrySubstitutableBy<_,_>` is not implemented for `{Self}`", label = "When trying to compress this parent type", - note = "#[da_compress(registry = \"...\")] was likely used on field with type {Self}" + note = "#[da_compress(registry)] was likely used on field with type {Self}" )] pub trait RegistrySubstitutableBy: Compressible where @@ -87,17 +87,17 @@ where { /// Perform substitution, returning the reference and possibly modifying the context. /// Typically the original value is stored into the context. - fn substitute(&self, keyspace: &str, ctx: &mut Ctx) -> Result; + fn substitute(&self, ctx: &mut Ctx) -> Result; } -/// Uses a decompression context +/// Uses a decompression context to desubstitute a type from a reference. /// This is used instead of `DecompressibleBy` when the type is desubstitutable from -/// a reference. Used with `da_compress(registry = "keyspace")` attribute from +/// a reference. Used with `da_compress(registry)` attribute from /// `fuel-derive::Compressed`. #[diagnostic::on_unimplemented( - message = "fuel_compression::RegistryDesubstitutableBy is not implemented for `{Self}`", + message = "`fuel_compression::RegistrySubstitutableBy<_,_>` is not implemented for `{Self}`", label = "When trying to decompress this parent type", - note = "#[da_compress(registry = \"...\")] was likely used on field with type {Self}" + note = "#[da_compress(registry)] was likely used on field with type {Self}" )] pub trait RegistryDesubstitutableBy: Compressible where @@ -106,5 +106,5 @@ where { /// Perform desubstitution, returning the original value. /// The context is typically used to resolve the reference. - fn desubstitute(c: &RawKey, keyspace: &str, ctx: &Ctx) -> Result; + fn desubstitute(c: &RawKey, ctx: &Ctx) -> Result; } diff --git a/fuel-derive/src/compressed.rs b/fuel-derive/src/compressed.rs index 6c0629f84e..2c1c7aaf45 100644 --- a/fuel-derive/src/compressed.rs +++ b/fuel-derive/src/compressed.rs @@ -101,16 +101,11 @@ pub enum FieldAttrs { /// Compresseded recursively. Normal, /// This value is compressed into a registry lookup. - /// `#[da_compress(registry = "keyspace")]` - Registry { - /// This is required to distinguish between values of the same type - keyspace: String, - }, + /// `#[da_compress(registry)]` + Registry, } impl FieldAttrs { pub fn parse(attrs: &[syn::Attribute]) -> Self { - let registry_path = syn::parse2::(quote! {registry}).unwrap(); - let mut result = Self::Normal; for attr in attrs { if attr.style != syn::AttrStyle::Outer { @@ -127,19 +122,9 @@ impl FieldAttrs { if ident == "skip" { result = Self::Skip; continue; - } - } else if let Ok(kv) = - syn::parse2::(ml.tokens.clone()) - { - if kv.path == registry_path { - if let syn::Expr::Lit(lit) = kv.value { - if let syn::Lit::Str(keyspace) = lit.lit { - result = Self::Registry { - keyspace: keyspace.value(), - }; - continue; - } - } + } else if ident == "registry" { + result = Self::Registry; + continue; } } panic!("Invalid attribute: {}", ml.tokens); @@ -170,7 +155,7 @@ fn field_defs(fields: &syn::Fields) -> TokenStream2 { quote! { #cty, } } } - FieldAttrs::Registry { .. } => { + FieldAttrs::Registry => { if let Some(fname) = field.ident.as_ref() { quote! { #fname: ::fuel_compression::RawKey, } } else { @@ -208,9 +193,9 @@ fn construct_compressed( let #cname = <#ty as ::fuel_compression::CompressibleBy<_, _>>::compress(&#binding, ctx)?; } } - FieldAttrs::Registry {keyspace} => { + FieldAttrs::Registry => { quote! { - let #cname = <#ty as ::fuel_compression::RegistrySubstitutableBy<_, _>>::substitute(&#binding, #keyspace, ctx)?; + let #cname = <#ty as ::fuel_compression::RegistrySubstitutableBy<_, _>>::substitute(&#binding, ctx)?; } } } @@ -268,9 +253,9 @@ fn construct_decompress( let #cname = <#ty as ::fuel_compression::DecompressibleBy<_, _>>::decompress(#binding, ctx)?; } } - FieldAttrs::Registry { keyspace } => { + FieldAttrs::Registry => { quote! { - let #cname = <#ty as ::fuel_compression::RegistryDesubstitutableBy<_, _>>::desubstitute(#binding, #keyspace, ctx)?; + let #cname = <#ty as ::fuel_compression::RegistryDesubstitutableBy<_, _>>::desubstitute(#binding, ctx)?; } } } @@ -404,7 +389,7 @@ pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { syn::parse_quote! { #ty: ::fuel_compression::CompressibleBy }, ); } - FieldAttrs::Registry { .. } => { + FieldAttrs::Registry => { where_clause_push( &mut w_impl_field_bounds_compress, syn::parse_quote! { #ty: ::fuel_compression::RegistrySubstitutableBy }, @@ -426,7 +411,7 @@ pub fn compressed_derive(mut s: synstructure::Structure) -> TokenStream2 { syn::parse_quote! { #ty: ::fuel_compression::DecompressibleBy }, ); } - FieldAttrs::Registry { .. } => { + FieldAttrs::Registry => { where_clause_push( &mut w_impl_field_bounds_decompress, syn::parse_quote! { #ty: ::fuel_compression::RegistryDesubstitutableBy }, diff --git a/fuel-tx/src/builder.rs b/fuel-tx/src/builder.rs index 6adb846424..67ebfe5a7a 100644 --- a/fuel-tx/src/builder.rs +++ b/fuel-tx/src/builder.rs @@ -131,7 +131,7 @@ impl TransactionBuilder