From bfdfce68a3155e7bd84f40d4bc70526bceffc3c3 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 22 Mar 2021 12:48:25 +0100 Subject: [PATCH 01/20] Transaction storage runtime module --- Cargo.lock | 34 ++ Cargo.toml | 2 + bin/node/runtime/Cargo.toml | 1 + bin/node/runtime/src/lib.rs | 10 + client/api/src/client.rs | 10 + client/api/src/in_mem.rs | 7 + client/api/src/lib.rs | 1 + client/db/src/lib.rs | 44 +- client/light/src/blockchain.rs | 7 + client/service/src/client/client.rs | 7 + frame/transaction-storage/Cargo.toml | 43 ++ frame/transaction-storage/README.md | 238 +++++++++++ frame/transaction-storage/src/lib.rs | 394 ++++++++++++++++++ primitives/blockchain/src/backend.rs | 2 + primitives/externalities/src/lib.rs | 4 +- primitives/io/src/lib.rs | 32 ++ primitives/state-machine/src/ext.rs | 17 +- .../src/overlayed_changes/mod.rs | 8 +- primitives/storage-proof/Cargo.toml | 39 ++ primitives/storage-proof/README.md | 3 + primitives/storage-proof/src/lib.rs | 242 +++++++++++ primitives/trie/src/storage_proof.rs | 4 + 22 files changed, 1128 insertions(+), 21 deletions(-) create mode 100644 frame/transaction-storage/Cargo.toml create mode 100644 frame/transaction-storage/README.md create mode 100644 frame/transaction-storage/src/lib.rs create mode 100644 primitives/storage-proof/Cargo.toml create mode 100644 primitives/storage-proof/README.md create mode 100644 primitives/storage-proof/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c8a7299835a06..ac63de633f69f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4377,6 +4377,7 @@ dependencies = [ "pallet-tips", "pallet-transaction-payment", "pallet-transaction-payment-rpc-runtime-api", + "pallet-transaction-storage", "pallet-treasury", "pallet-uniques", "pallet-utility", @@ -5602,6 +5603,24 @@ dependencies = [ "sp-runtime", ] +[[package]] +name = "pallet-transaction-storage" +version = "3.0.0" +dependencies = [ + "frame-support", + "frame-support-test", + "frame-system", + "pallet-balances", + "parity-scale-codec", + "serde", + "sp-core", + "sp-inherents", + "sp-io", + "sp-runtime", + "sp-std", + "sp-storage-proof", +] + [[package]] name = "pallet-treasury" version = "3.0.0" @@ -9241,6 +9260,21 @@ dependencies = [ "sp-std", ] +[[package]] +name = "sp-storage-proof" +version = "3.0.0" +dependencies = [ + "log", + "parity-scale-codec", + "sc-client-api", + "sp-consensus", + "sp-core", + "sp-inherents", + "sp-runtime", + "sp-std", + "sp-trie", +] + [[package]] name = "sp-tasks" version = "3.0.0" diff --git a/Cargo.toml b/Cargo.toml index 8b613c021a9fe..02daa6e5ce6a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,7 @@ members = [ "frame/transaction-payment", "frame/transaction-payment/rpc", "frame/transaction-payment/rpc/runtime-api", + "frame/transaction-storage", "frame/treasury", "frame/tips", "frame/uniques", @@ -175,6 +176,7 @@ members = [ "primitives/state-machine", "primitives/std", "primitives/storage", + "primitives/storage-proof", "primitives/tasks", "primitives/test-primitives", "primitives/timestamp", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index ca1ed7f3dcc09..d53dc4013e77e 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -85,6 +85,7 @@ pallet-treasury = { version = "3.0.0", default-features = false, path = "../../. pallet-utility = { version = "3.0.0", default-features = false, path = "../../../frame/utility" } pallet-transaction-payment = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-payment" } pallet-transaction-payment-rpc-runtime-api = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } +pallet-transaction-storage = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-storage" } pallet-uniques = { version = "3.0.0", default-features = false, path = "../../../frame/uniques" } pallet-vesting = { version = "3.0.0", default-features = false, path = "../../../frame/vesting" } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 3732adfb9a78d..37b439cacea71 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1114,6 +1114,12 @@ impl pallet_uniques::Config for Runtime { type WeightInfo = pallet_uniques::weights::SubstrateWeight; } +impl pallet_transaction_storage::Config for Runtime { + type Event = Event; + type Currency = Balances; + type Call = Call; +} + construct_runtime!( pub enum Runtime where Block = Block, @@ -1158,7 +1164,11 @@ construct_runtime!( Mmr: pallet_mmr::{Pallet, Storage}, Lottery: pallet_lottery::{Pallet, Call, Storage, Event}, Gilt: pallet_gilt::{Pallet, Call, Storage, Event, Config}, +<<<<<<< HEAD Uniques: pallet_uniques::{Pallet, Call, Storage, Event}, +======= + TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Event}, +>>>>>>> 589fb673e6 (Transaction storage runtime module) } ); diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 4a0940b1f4bd3..d5cb43931dca4 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -109,6 +109,16 @@ pub trait BlockBackend { fn has_indexed_transaction(&self, hash: &Block::Hash) -> sp_blockchain::Result { Ok(self.indexed_transaction(hash)?.is_some()) } + + /// Get all indexed transactions for a block, + /// including renewed transactions. + /// + /// Note that this will only fetch transactions + /// that are indexed by the runtime with `storage_index_transaction`. + fn block_indexed_body( + &self, + id: &BlockId, + ) -> sp_blockchain::Result>>>; } /// Provide a list of potential uncle headers for a given block. diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index d756e1cc0bbc4..0d40bb3354cc3 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -419,6 +419,13 @@ impl blockchain::Backend for Blockchain { ) -> sp_blockchain::Result>> { unimplemented!("Not supported by the in-mem backend.") } + + fn block_indexed_body( + &self, + _id: BlockId + ) -> sp_blockchain::Result>>> { + unimplemented!("Not supported by the in-mem backend.") + } } impl blockchain::ProvideCache for Blockchain { diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index 0f860b95e7805..f3cef0e36ff47 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -38,6 +38,7 @@ pub use client::*; pub use light::*; pub use notifications::*; pub use proof_provider::*; +pub use sp_blockchain::HeaderBackend; pub use sp_state_machine::{StorageProof, ExecutionStrategy}; diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 9a334f95d49a1..d3e451dc49af7 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -67,7 +67,7 @@ use codec::{Decode, Encode}; use hash_db::Prefix; use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; use sp_database::Transaction; -use sp_core::{Hasher, ChangesTrieConfiguration}; +use sp_core::ChangesTrieConfiguration; use sp_core::offchain::OffchainOverlayedChange; use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; @@ -591,6 +591,37 @@ impl sc_client_api::blockchain::Backend for BlockchainDb ClientResult { Ok(self.db.contains(columns::TRANSACTION, hash.as_ref())) } + + fn block_indexed_body(&self, id: BlockId) -> ClientResult>>> { + match self.transaction_storage { + TransactionStorageMode::BlockBody => Ok(None), + TransactionStorageMode::StorageChain => { + let body = match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? { + Some(body) => body, + None => return Ok(None), + }; + match Vec::::decode(&mut &body[..]) { + Ok(index) => { + let mut transactions = Vec::new(); + for ExtrinsicHeader { indexed_hash, .. } in index.into_iter() { + if indexed_hash != Default::default() { + match self.db.get(columns::TRANSACTION, indexed_hash.as_ref()) { + Some(t) => transactions.push(t), + None => return Err(sp_blockchain::Error::Backend( + format!("Missing indexed transaction {:?}", indexed_hash)) + ) + } + } + } + Ok(Some(transactions)) + } + Err(err) => return Err(sp_blockchain::Error::Backend( + format!("Error decoding body list: {}", err) + )), + } + } + } + } } impl sc_client_api::blockchain::ProvideCache for BlockchainDb { @@ -1624,10 +1655,10 @@ fn apply_index_ops( let mut renewed_map = HashMap::new(); for op in ops { match op { - IndexOperation::Insert { extrinsic, offset } => { - index_map.insert(extrinsic, offset); + IndexOperation::Insert { extrinsic, hash, size } => { + index_map.insert(extrinsic, (hash, size)); } - IndexOperation::Renew { extrinsic, hash, .. } => { + IndexOperation::Renew { extrinsic, hash } => { renewed_map.insert(extrinsic, DbHash::from_slice(hash.as_ref())); } } @@ -1643,9 +1674,8 @@ fn apply_index_ops( } } else { match index_map.get(&(index as u32)) { - Some(offset) if *offset as usize <= extrinsic.len() => { - let offset = *offset as usize; - let hash = HashFor::::hash(&extrinsic[offset..]); + Some((hash, size)) if *size as usize <= extrinsic.len() => { + let offset = extrinsic.len() - *size as usize; transaction.store( columns::TRANSACTION, DbHash::from_slice(hash.as_ref()), diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs index 3349adf7ac693..242839833a541 100644 --- a/client/light/src/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -135,6 +135,13 @@ impl BlockchainBackend for Blockchain where Block: BlockT, S ) -> ClientResult>> { Err(ClientError::NotAvailableOnLightClient) } + + fn block_indexed_body( + &self, + _id: BlockId + ) -> sp_blockchain::Result>>> { + Err(ClientError::NotAvailableOnLightClient) + } } impl, Block: BlockT> ProvideCache for Blockchain { diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index b294be2268997..b21d6b31aaf89 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1982,6 +1982,13 @@ impl BlockBackend for Client fn has_indexed_transaction(&self, hash: &Block::Hash) -> sp_blockchain::Result { self.backend.blockchain().has_indexed_transaction(hash) } + + fn block_indexed_body( + &self, + id: &BlockId + ) -> sp_blockchain::Result>>> { + self.backend.blockchain().block_indexed_body(*id) + } } impl backend::AuxStore for Client diff --git a/frame/transaction-storage/Cargo.toml b/frame/transaction-storage/Cargo.toml new file mode 100644 index 0000000000000..863408dc1ac52 --- /dev/null +++ b/frame/transaction-storage/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "pallet-transaction-storage" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Storage chain pallet" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +frame-support = { version = "3.0.0", default-features = false, path = "../support" } +frame-system = { version = "3.0.0", default-features = false, path = "../system" } +pallet-balances = { version = "3.0.0", default-features = false, path = "../balances" } +sp-runtime = { version = "3.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "3.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "3.0.0", default-features = false, path = "../../primitives/io" } +sp-inherents = { version = "3.0.0", default-features = false, path = "../../primitives/inherents" } +sp-storage-proof = { version = "3.0.0", default-features = false, path = "../../primitives/storage-proof" } + +[dev-dependencies] +sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false } +frame-support-test = { version = "3.0.0", path = "../support/test" } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "sp-runtime/std", + "frame-support/std", + "frame-system/std", + "pallet-balances/std", + "sp-io/std", + "sp-std/std", + "sp-inherents/std", +] diff --git a/frame/transaction-storage/README.md b/frame/transaction-storage/README.md new file mode 100644 index 0000000000000..46a0d076a969a --- /dev/null +++ b/frame/transaction-storage/README.md @@ -0,0 +1,238 @@ + +# Example Pallet + + +The Example: A simple example of a FRAME pallet demonstrating +concepts, APIs and structures common to most FRAME runtimes. + +Run `cargo doc --package pallet-example --open` to view this pallet's documentation. + +### Documentation Guidelines: + + + +
    +
  • Documentation comments (i.e. /// comment) - should + accompany pallet functions and be restricted to the pallet interface, + not the internals of the pallet implementation. Only state inputs, + outputs, and a brief description that mentions whether calling it + requires root, but without repeating the source code details. + Capitalize the first word of each documentation comment and end it with + a full stop. See + Generic example of annotating source code with documentation comments
    • +
  • Self-documenting code - Try to refactor code to be self-documenting.
    • +
  • Code comments - Supplement complex code with a brief explanation, not every line of code.
    • +
  • Identifiers - surround by backticks (i.e. INHERENT_IDENTIFIER, InherentType, + u64)
    • +
  • Usage scenarios - should be simple doctests. The compiler should ensure they stay valid.
    • +
  • Extended tutorials - should be moved to external files and refer to.
    • + +
  • Mandatory - include all of the sections/subsections where MUST is specified.
    • +
  • Optional - optionally include sections/subsections where CAN is specified.
    • +
+ +### Documentation Template:
+ +Copy and paste this template from frame/example/src/lib.rs into file +`frame//src/lib.rs` of your own custom pallet and complete it. +

+// Add heading with custom pallet name
+
+\#  Pallet
+
+// Add simple description
+
+// Include the following links that shows what trait needs to be implemented to use the pallet
+// and the supported dispatchables that are documented in the Call enum.
+
+- \[`::Trait`](https://docs.rs/pallet-example/latest/pallet_example/trait.Trait.html)
+- \[`Call`](https://docs.rs/pallet-example/latest/pallet_example/enum.Call.html)
+- \[`Module`](https://docs.rs/pallet-example/latest/pallet_example/struct.Module.html)
+
+\## Overview
+
+
+// Short description of pallet's purpose.
+// Links to Traits that should be implemented.
+// What this pallet is for.
+// What functionality the pallet provides.
+// When to use the pallet (use case examples).
+// How it is used.
+// Inputs it uses and the source of each input.
+// Outputs it produces.
+
+
+
+
+\## Terminology
+
+// Add terminology used in the custom pallet. Include concepts, storage items, or actions that you think
+// deserve to be noted to give context to the rest of the documentation or pallet usage. The author needs to
+// use some judgment about what is included. We don't want a list of every storage item nor types - the user
+// can go to the code for that. For example, "transfer fee" is obvious and should not be included, but
+// "free balance" and "reserved balance" should be noted to give context to the pallet.
+// Please do not link to outside resources. The reference docs should be the ultimate source of truth.
+
+
+
+\## Goals
+
+// Add goals that the custom pallet is designed to achieve.
+
+
+
+\### Scenarios
+
+
+
+\#### 
+
+// Describe requirements prior to interacting with the custom pallet.
+// Describe the process of interacting with the custom pallet for this scenario and public API functions used.
+
+\## Interface
+
+\### Supported Origins
+
+// What origins are used and supported in this pallet (root, signed, none)
+// i.e. root when \`ensure_root\` used
+// i.e. none when \`ensure_none\` used
+// i.e. signed when \`ensure_signed\` used
+
+\`inherent\` 
+
+
+
+
+\### Types
+
+// Type aliases. Include any associated types and where the user would typically define them.
+
+\`ExampleType\` 
+
+
+
+// Reference documentation of aspects such as `storageItems` and `dispatchable` functions should only be
+// included in the https://docs.rs Rustdocs for Substrate and not repeated in the README file.
+
+\### Dispatchable Functions
+
+
+
+// A brief description of dispatchable functions and a link to the rustdoc with their actual documentation.
+
+// MUST have link to Call enum
+// MUST have origin information included in function doc
+// CAN have more info up to the user
+
+\### Public Functions
+
+
+
+// A link to the rustdoc and any notes about usage in the pallet, not for specific functions.
+// For example, in the Balances Pallet: "Note that when using the publicly exposed functions,
+// you (the runtime developer) are responsible for implementing any necessary checks
+// (e.g. that the sender is the signer) before calling a function that will affect storage."
+
+
+
+// It is up to the writer of the respective pallet (with respect to how much information to provide).
+
+\#### Public Inspection functions - Immutable (getters)
+
+// Insert a subheading for each getter function signature
+
+\##### \`example_getter_name()\`
+
+// What it returns
+// Why, when, and how often to call it
+// When it could panic or error
+// When safety issues to consider
+
+\#### Public Mutable functions (changing state)
+
+// Insert a subheading for each setter function signature
+
+\##### \`example_setter_name(origin, parameter_name: T::ExampleType)\`
+
+// What state it changes
+// Why, when, and how often to call it
+// When it could panic or error
+// When safety issues to consider
+// What parameter values are valid and why
+
+\### Storage Items
+
+// Explain any storage items included in this pallet
+
+\### Digest Items
+
+// Explain any digest items included in this pallet
+
+\### Inherent Data
+
+// Explain what inherent data (if any) is defined in the pallet and any other related types
+
+\### Events:
+
+// Insert events for this pallet if any
+
+\### Errors:
+
+// Explain what generates errors
+
+\## Usage
+
+// Insert 2-3 examples of usage and code snippets that show how to
+// use  Pallet in a custom pallet.
+
+\### Prerequisites
+
+// Show how to include necessary imports for  and derive
+// your pallet configuration trait with the `INSERT_CUSTOM_PALLET_NAME` trait.
+
+\```rust
+use ;
+
+pub trait Config: ::Config { }
+\```
+
+\### Simple Code Snippet
+
+// Show a simple example (e.g. how to query a public getter function of )
+
+\### Example from FRAME
+
+// Show a usage example in an actual runtime
+
+// See:
+// - Substrate TCR https://github.com/parity-samples/substrate-tcr
+// - Substrate Kitties https://shawntabrizi.github.io/substrate-collectables-workshop/#/
+
+\## Genesis Config
+
+
+
+\## Dependencies
+
+// Dependencies on other FRAME pallets and the genesis config should be mentioned,
+// but not the Rust Standard Library.
+// Genesis configuration modifications that may be made to incorporate this pallet
+// Interaction with other pallets
+
+
+
+\## Related Pallets
+
+// Interaction with other pallets in the form of a bullet point list
+
+\## References
+
+
+
+// Links to reference material, if applicable. For example, Phragmen, W3F research, etc.
+// that the implementation is based on.
+

+ +License: Unlicense diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs new file mode 100644 index 0000000000000..54a597b439a86 --- /dev/null +++ b/frame/transaction-storage/src/lib.rs @@ -0,0 +1,394 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Transaction storage pallet. Indexes transactions and manages storage proofs. + + +// Ensure we're `no_std` when compiling for Wasm. +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::{ + traits::{ReservableCurrency, Currency}, + dispatch::{Dispatchable, GetDispatchInfo}, +}; +use sp_std::prelude::*; +use sp_std::{result}; +use codec::{Encode, Decode}; +use sp_runtime::{ + traits::{ + Saturating, BlakeTwo256, Hash, Zero, + }, +}; +use sp_storage_proof::StorageProof; + +/// A type alias for the balance type from this pallet's point of view. +type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +// Re-export pallet items so that they can be accessed from the crate namespace. +pub use pallet::*; + +const CHUNK_SIZE: usize = sp_storage_proof::CHUNK_SIZE; + +#[derive(Encode, Decode, sp_runtime::RuntimeDebug)] +pub struct TransactionInfo { + chunk_root: ::Output, + content_hash: ::Output, + size: u32, +} + +// Definition of the pallet logic, to be aggregated at runtime definition through +// `construct_runtime`. +#[frame_support::pallet] +pub mod pallet { + // Import various types used to declare pallet in scope. + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + use super::*; + + /// Our pallet's configuration trait. All our types and constants go in here. If the + /// pallet is dependent on specific other pallets, then their configuration traits + /// should be added to our implied traits list. + /// + /// `frame_system::Config` should always be included. + #[pallet::config] + pub trait Config: pallet_balances::Config + frame_system::Config { + /// The overarching event type. + type Event: From> + IsType<::Event>; + /// A dispatchable call. + type Call: Parameter + Dispatchable + GetDispatchInfo + From>; + /// The currency trait. + type Currency: ReservableCurrency; + } + + #[pallet::error] + pub enum Error { + /// Insufficient account balance. + InsufficientFunds, + /// Invalid configuration. + NotConfigured, + /// Renewed extrinsic is not found. + RenewedNotFound, + /// Attempting to store empty transaction + EmptyTransaction, + /// Proof was not expected in this block. + UnexpectedProof, + /// Proof failed verification. + InvalidProof, + /// Missing storage proof. + MissingProof, + /// Unable to verify proof becasue state data is missing. + MissingStateData, + } + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(_n: T::BlockNumber) -> Weight { + >::put(0); + 0 + } + + fn on_finalize(n: T::BlockNumber) { + >::insert(n, >::get().unwrap_or(0)); + // Drop obsolete roots + >::kill(); + let period = >::get().unwrap(); + let obsolete = n.saturating_sub(period); + if obsolete > Zero::zero() { + >::remove_prefix(obsolete); + >::remove(obsolete); + } + } + } + + #[pallet::call] + impl Pallet { + #[pallet::weight(0)] + pub(super) fn store( + origin: OriginFor, + data: Vec, + ) -> DispatchResultWithPostInfo { + ensure!(data.len() > 0, Error::::EmptyTransaction); + Self::apply_fee(origin, data.len() as u32)?; + + // Chunk data and compute storage root + let chunks = data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); + let root = sp_io::trie::blake2_256_ordered_root(chunks); + + let content_hash = sp_io::hashing::blake2_256(&data); + let extrinsic_index = >::extrinsic_index().unwrap(); + sp_io::transaction_index::index(extrinsic_index, data.len() as u32, content_hash); + + let block_num = >::block_number(); + let mut counter = >::get().unwrap_or(0); + >::insert(block_num, counter, TransactionInfo { + chunk_root: root, + size: data.len() as u32, + content_hash: content_hash.into(), + }); + counter += 1; + >::put(counter); + + Self::deposit_event(Event::Stored); + Ok(().into()) + } + + #[pallet::weight(0)] + pub(super) fn renew( + origin: OriginFor, + block: T::BlockNumber, + index: u32, + ) -> DispatchResultWithPostInfo { + let info = >::get(block, index).ok_or(Error::::RenewedNotFound)?; + Self::apply_fee(origin, info.size)?; + + let extrinsic_index = >::extrinsic_index().unwrap(); + sp_io::transaction_index::renew(extrinsic_index, info.content_hash.into()); + + let mut counter = >::get().unwrap_or(0); + let block_num = >::block_number(); + >::insert(block_num, counter, info); + counter += 1; + >::put(counter); + + Self::deposit_event(Event::Renewed); + Ok(().into()) + } + + #[pallet::weight(0)] + fn check_proof( + origin: OriginFor, + proof: Option, + ) -> DispatchResultWithPostInfo { + ensure_none(origin)?; + let number = >::block_number(); + let period = >::get().unwrap(); + let target_number = number.saturating_sub(period); + if target_number.is_zero() { + ensure!(proof.is_none(), Error::::UnexpectedProof); + return Ok(().into()) + } + let transaction_count = >::get(target_number); + let mut total_chunks = 0; + for t in 0 .. transaction_count { + match >::get(target_number, t) { + Some(info) => total_chunks += ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32, + None => break, + } + } + if total_chunks == 0 { + ensure!(proof.is_none(), Error::::UnexpectedProof); + return Ok(().into()) + } + let proof = proof.ok_or_else(|| Error::::MissingProof)?; + let parent_hash = >::parent_hash(); + let selected_chunk_index = sp_storage_proof::random_chunk(parent_hash.as_ref(), total_chunks); + total_chunks = 0; + let mut t = 0; + let (info, chunk_index) = loop { + match >::get(target_number, t) { + Some(info) => { + total_chunks += ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32; + if total_chunks >= selected_chunk_index { + break (info, total_chunks - selected_chunk_index) + } + }, + None => Err(Error::::MissingStateData)?, + } + t += 1; + }; + + ensure!( + sp_io::trie::blake2_256_verity_proof( + info.chunk_root, + &proof.proof, + &(chunk_index as u32).to_be_bytes(), + &proof.chunk, + ), + Error::::InvalidProof + ); + Ok(().into()) + } + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + Stored, + Renewed, + } + + /// Collection of extrinsic storage roots for the current block. + #[pallet::storage] + #[pallet::getter(fn transaction_roots)] + pub(super) type TransactionRoots = StorageDoubleMap< + _, + Blake2_128Concat, + T::BlockNumber, + Blake2_128Concat, + u32, + TransactionInfo, + OptionQuery, + >; + + /// Count indexed chunks for each block. + #[pallet::storage] + pub(super) type TransactionCount = StorageMap< + _, + Blake2_128Concat, + T::BlockNumber, + u32, + ValueQuery, + >; + + #[pallet::storage] + #[pallet::getter(fn byte_fee)] + pub(super) type ByteFee = StorageValue<_, BalanceOf>; + + #[pallet::storage] + pub(super) type Counter = StorageValue<_, u32>; + + #[pallet::storage] + pub(super) type StoragePeriod = StorageValue<_, T::BlockNumber>; + + #[pallet::inherent] + impl ProvideInherent for Pallet { + type Call = Call; + type Error = sp_storage_proof::InherentError; + const INHERENT_IDENTIFIER: InherentIdentifier = sp_storage_proof::INHERENT_IDENTIFIER; + + fn create_inherent(data: &InherentData) -> Option { + let proof = data.get_data::(&Self::INHERENT_IDENTIFIER).unwrap_or(None); + Some(Call::check_proof(proof)) + } + + fn check_inherent(_call: &Self::Call, _data: &InherentData) -> result::Result<(), Self::Error> { + Ok(()) + } + } + + impl Pallet { + fn apply_fee(origin: OriginFor, size: u32) -> DispatchResult { + let sender = ensure_signed(origin)?; + let byte_fee = ByteFee::::get().ok_or(Error::::NotConfigured)?; + let fee = byte_fee.saturating_mul(size.into()); + ensure!(T::Currency::can_slash(&sender, fee), Error::::InsufficientFunds); + T::Currency::slash(&sender, fee); + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use frame_support::{ + assert_ok, parameter_types, + weights::{DispatchInfo, GetDispatchInfo}, traits::{OnInitialize, OnFinalize} + }; + use sp_core::H256; + // The testing primitives are very useful for avoiding having to work with signatures + // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. + use sp_runtime::{ + testing::Header, BuildStorage, + traits::{BlakeTwo256, IdentityLookup}, + }; + // Reexport crate as its pallet name for construct_runtime. + use crate as pallet_transaction_storage; + + type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; + type Block = frame_system::mocking::MockBlock; + + // For testing the pallet, we construct a mock runtime. + frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, + TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Event}, + } + ); + + parameter_types! { + pub const BlockHashCount: u64 = 250; + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::simple_max(1024); + } + impl frame_system::Config for Test { + type BaseCallFilter = (); + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Call = Call; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + } + parameter_types! { + pub const ExistentialDeposit: u64 = 1; + } + impl pallet_balances::Config for Test { + type MaxLocks = (); + type Balance = u64; + type DustRemoval = (); + type Event = Event; + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type WeightInfo = (); + } + impl Config for Test { + type Event = Event; + type Call = Call; + type Currency = Balances; + } + + // This function basically just builds a genesis storage key/value store according to + // our desired mockup. + pub fn new_test_ext() -> sp_io::TestExternalities { + let t = GenesisConfig { + // We use default for brevity, but you can configure as desired if needed. + frame_system: Default::default(), + pallet_balances: Default::default(), + }.build_storage().unwrap(); + t.into() + } + + #[test] + fn weights_work() { + } +} diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index b00cbada9f476..3441a4f6cf544 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -227,6 +227,8 @@ pub trait Backend: HeaderBackend + HeaderMetadata Result { Ok(self.indexed_transaction(hash)?.is_some()) } + + fn block_indexed_body(&self, id: BlockId) -> Result>>>; } /// Provides access to the optional cache. diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index ce5a0990d738d..14145e8798498 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -229,12 +229,12 @@ pub trait Externalities: ExtensionStore { fn storage_commit_transaction(&mut self) -> Result<(), ()>; /// Index specified transaction slice and store it. - fn storage_index_transaction(&mut self, _index: u32, _offset: u32) { + fn storage_index_transaction(&mut self, _index: u32, _hash: &[u8], _size: u32) { unimplemented!("storage_index_transaction"); } /// Renew existing piece of transaction storage. - fn storage_renew_transaction_index(&mut self, _index: u32, _hash: &[u8], _size: u32) { + fn storage_renew_transaction_index(&mut self, _index: u32, _hash: &[u8]) { unimplemented!("storage_renew_transaction_index"); } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 72695f2156b67..08902e9ebc9f3 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -429,6 +429,24 @@ pub trait Trie { fn keccak_256_ordered_root(input: Vec>) -> H256 { Layout::::ordered_trie_root(input) } + + /// Verify trie proof + fn blake2_256_verity_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { + sp_trie::verify_trie_proof::, _, _, _>( + &root, + proof, + &[(key, Some(value))], + ).is_ok() + } + + /// Verify trie proof + fn keccak_256_verity_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { + sp_trie::verify_trie_proof::, _, _, _>( + &root, + proof, + &[(key, Some(value))], + ).is_ok() + } } /// Interface that provides miscellaneous functions for communicating between the runtime and the node. @@ -824,6 +842,20 @@ pub trait Hashing { } } +/// Interface that provides transaction indexing API. +#[runtime_interface] +pub trait TransactionIndex { + /// Add transaction index. Returns indexed content hash. + fn index(&mut self, extrinsic: u32, size: u32, context_hash: [u8; 32]) { + self.storage_index_transaction(extrinsic, &context_hash, size); + } + + /// Conduct a 512-bit Keccak hash. + fn renew(&mut self, extrinsic: u32, context_hash: [u8; 32]) { + self.storage_renew_transaction_index(extrinsic, &context_hash); + } +} + /// Interface that provides functions to access the Offchain DB. #[runtime_interface] pub trait OffchainIndex { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 2649c320e14dc..8bcf1f28a0778 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -629,33 +629,34 @@ where } } - fn storage_index_transaction(&mut self, index: u32, offset: u32) { + fn storage_index_transaction(&mut self, index: u32, hash: &[u8], size: u32) { trace!( target: "state", - "{:04x}: IndexTransaction ({}): [{}..]", + "{:04x}: IndexTransaction ({}): {}, {} bytes", self.id, index, - offset, + HexDisplay::from(&hash), + size, ); self.overlay.add_transaction_index(IndexOperation::Insert { extrinsic: index, - offset, + hash: hash.to_vec(), + size, }); } /// Renew existing piece of data storage. - fn storage_renew_transaction_index(&mut self, index: u32, hash: &[u8], size: u32) { + fn storage_renew_transaction_index(&mut self, index: u32, hash: &[u8]) { trace!( target: "state", - "{:04x}: RenewTransactionIndex ({}) {} bytes", + "{:04x}: RenewTransactionIndex ({}): {}", self.id, + index, HexDisplay::from(&hash), - size, ); self.overlay.add_transaction_index(IndexOperation::Renew { extrinsic: index, hash: hash.to_vec(), - size }); } diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 2a3495a4e1c74..2fe5947a0c798 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -118,8 +118,10 @@ pub enum IndexOperation { Insert { /// Extrinsic index in the current block. extrinsic: u32, - /// Data offset in the extrinsic. - offset: u32, + /// Data content hash. + hash: Vec, + /// Indexed data size. + size: u32, }, /// Renew existing transaction storage. Renew { @@ -127,8 +129,6 @@ pub enum IndexOperation { extrinsic: u32, /// Referenced index hash. hash: Vec, - /// Expected data size. - size: u32, } } diff --git a/primitives/storage-proof/Cargo.toml b/primitives/storage-proof/Cargo.toml new file mode 100644 index 0000000000000..e5fc1c3f725a4 --- /dev/null +++ b/primitives/storage-proof/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "sp-storage-proof" +version = "3.0.0" +authors = ["Parity Technologies "] +description = "Storage proof primitives" +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sp-inherents = { version = "3.0.0", default-features = false, path = "../inherents" } +sp-runtime = { version = "3.0.0", default-features = false, path = "../runtime" } +sp-std = { version = "3.0.0", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } + +sc-client-api = { version = "3.0.0", path = "../../client/api", optional = true } +sp-consensus = { version = "0.9.0", path = "../consensus/common", optional = true } +sp-trie = { version = "3.0.0", optional = true, path = "../trie" } +sp-core = { version = "3.0.0", path = "../core", optional = true } +log = { version = "0.4.8", optional = true } + +[features] +default = [ "std" ] +std = [ + "codec/std", + "sp-std/std", + "sp-inherents/std", + "sp-runtime/std", + "sp-trie/std", + "sp-consensus", + "sp-core", + "sc-client-api", + "log", +] diff --git a/primitives/storage-proof/README.md b/primitives/storage-proof/README.md new file mode 100644 index 0000000000000..1aa1805cfc5e7 --- /dev/null +++ b/primitives/storage-proof/README.md @@ -0,0 +1,3 @@ +Authorship Primitives + +License: Apache-2.0 \ No newline at end of file diff --git a/primitives/storage-proof/src/lib.rs b/primitives/storage-proof/src/lib.rs new file mode 100644 index 0000000000000..808544d4fb641 --- /dev/null +++ b/primitives/storage-proof/src/lib.rs @@ -0,0 +1,242 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storge Proof Primitives + +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::{result::Result, prelude::*}; + +use codec::{Encode, Decode}; +use sp_inherents::{Error, InherentIdentifier, InherentData, IsFatalError}; + +/// The identifier for the proof inherent. +pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"tx_proof"; +pub const STORAGE_PERIOD: u32 = 10; +pub const CHUNK_SIZE: usize = 256; + +/// Errors that can occur while checking the storage proof. +#[derive(Encode, sp_runtime::RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Decode))] +pub enum InherentError { + InvalidProof, + TrieError +} + +impl IsFatalError for InherentError { + fn is_fatal_error(&self) -> bool { + true + } +} + +#[derive(Encode, Decode, Clone, PartialEq, Debug)] +pub struct StorageProof { + /// Data chunk that is proved to exist. + pub chunk: Vec, + /// Trie nodes that compose the proof. + pub proof: Vec>, +} + +/// Auxiliary trait to extract storage proof. +pub trait StorageProofInherentData { + /// Get the proof. + fn storage_proof(&self) -> Result, Error>; +} + +impl StorageProofInherentData for InherentData { + fn storage_proof(&self) -> Result, Error> { + Ok(self.get_data(&INHERENT_IDENTIFIER)?) + } +} + +/// Provider for inherent data. +#[cfg(feature = "std")] +pub struct InherentDataProvider { + inner: F, +} + +#[cfg(feature = "std")] +impl InherentDataProvider { + pub fn new(proof_oracle: F) -> Self { + InherentDataProvider { inner: proof_oracle } + } +} + +#[cfg(feature = "std")] +impl sp_inherents::ProvideInherentData for InherentDataProvider +where F: Fn() -> Result, Error> +{ + fn inherent_identifier(&self) -> &'static InherentIdentifier { + &INHERENT_IDENTIFIER + } + + fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), Error> { + let proof = (self.inner)()?; + if let Some(proof) = proof { + inherent_data.put_data(INHERENT_IDENTIFIER, &proof) + } else { + Ok(()) + } + } + + fn error_to_string(&self, _error: &[u8]) -> Option { + Some(format!("no further information")) + } +} + + +pub fn random_chunk(random_hash: &[u8], total_chunks: u32) -> u32 { + let mut buf = [0u8; 8]; + buf.copy_from_slice(&random_hash[0..8]); + let random_u64 = u64::from_be_bytes(buf); + (random_u64 % total_chunks as u64) as u32 +} + +#[cfg(feature = "std")] +pub mod registration { + use sp_consensus::SelectChain; + use sp_inherents::{InherentDataProviders}; + use log::warn; + use sc_client_api::{HeaderBackend, BlockBackend}; + use sp_runtime::{traits::{Block as BlockT, Header, Saturating, Zero}, generic::BlockId}; + use std::sync::Arc; + use sp_trie::{TrieConfiguration, TrieMut}; + use crate::{StorageProof, CHUNK_SIZE}; + use std::iter::once; + + type TrieLayout = sp_trie::Layout::; + type Hasher = sp_core::Blake2Hasher; + + + /// Register uncles inherent data provider, if not registered already. + pub fn register_storage_proof_inherent_data_provider( + client: Arc, + select_chain: SC, + inherent_data_providers: &InherentDataProviders, + ) -> Result<(), sp_consensus::Error> where + B: BlockT, + C: BlockBackend + HeaderBackend + Send + Sync + 'static, + SC: SelectChain + 'static, + { + if !inherent_data_providers.has_provider(&crate::INHERENT_IDENTIFIER) { + inherent_data_providers + .register_provider(crate::InherentDataProvider::new(move || { + { + let chain_head = match select_chain.best_chain() { + Ok(x) => x, + Err(e) => { + warn!(target: "storage-proof", "Unable to get chain head: {:?}", e); + return Ok(None); + } + }; + + let number = chain_head.number().saturating_sub(crate::STORAGE_PERIOD.into()); + if number.is_zero() { + // Too early to collect proofs. + return Ok(None); + } + + match client.block_indexed_body(&BlockId::number(number)) { + Ok(Some(transactions)) => { + Ok(Some(build_proof::(chain_head.parent_hash().as_ref(), transactions)?)) + }, + Ok(None) => { + // Nothing was indexed in that block. + Ok(None) + } + Err(e) => { + warn!(target: "storage-proof", "Unable to get transactions: {:?}", e); + Ok(None) + } + } + } + })) + .map_err(|err| sp_consensus::Error::InherentData(err.into()))?; + } + Ok(()) + } + + fn build_proof(random_hash: &[u8], transactions: Vec>) + -> Result + { + let mut block_root = sp_trie::empty_trie_root::(); + let mut db = sp_trie::MemoryDB::::default(); + + let mut target_chunk = None; + let mut target_root = Default::default(); + let mut target_chunk_key = Default::default(); + let mut target_block_key = Default::default(); + let mut chunk_proof = Default::default(); + + let total_chunks: u64 = transactions.iter().map(|t| ((t.len() + CHUNK_SIZE - 1) / CHUNK_SIZE) as u64).sum(); + let mut buf = [0u8; 8]; + buf.copy_from_slice(&random_hash[0..8]); + let random_u64 = u64::from_be_bytes(buf); + let target_chunk_index = random_u64 % total_chunks; + //Generate tries for each + + let mut chunk_index = 0; + for (ti, transaction) in transactions.into_iter().enumerate() { + let mut transaction_root = sp_trie::empty_trie_root::(); + { + let mut trie = sp_trie::TrieDBMut::::new(&mut db, &mut transaction_root); + let chunks = transaction.chunks(crate::CHUNK_SIZE).map(|c| c.to_vec()); + for (index, chunk) in chunks.enumerate() { + let index = TrieLayout::encode_index(index as u32); + trie.insert(&index, &chunk) + .map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; + if chunk_index == target_chunk_index { + target_chunk = Some(chunk); + target_chunk_key = index; + } + chunk_index += 1; + } + trie.commit(); + } + let transaction_key = TrieLayout::encode_index(ti as u32); + if target_chunk.is_some() && target_root == Default::default() { + target_root = transaction_root.clone(); + target_block_key = transaction_key.clone(); + chunk_proof = sp_trie::generate_trie_proof::( + &db, + transaction_root.clone(), + &[target_chunk_key.clone()] + ).map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; + } + { + let mut block_trie = sp_trie::TrieDBMut::::new(&mut db, &mut block_root); + block_trie.insert(&transaction_key, transaction_root.as_ref()) + .map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; + block_trie.commit(); + } + }; + let block_proof = sp_trie::generate_trie_proof::( + &db, + block_root, + &[target_block_key] + ).map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; + + let proof = sp_trie::StorageProof::merge( + once(sp_trie::StorageProof::new(chunk_proof)).chain( + once(sp_trie::StorageProof::new(block_proof)))); + + Ok(StorageProof { + proof: proof.into_nodes(), + chunk: target_chunk.unwrap(), + }) + } +} diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index f0b2bfd4bc3d3..d8394a89de526 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -58,6 +58,10 @@ impl StorageProof { StorageProofNodeIterator::new(self) } + /// Convert into plain node vector. + pub fn into_nodes(self) -> Vec> { + self.trie_nodes + } /// Creates a `MemoryDB` from `Self`. pub fn into_memory_db(self) -> crate::MemoryDB { self.into() From 178493593d441291b970891d5db5d1e3ae6774eb Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 25 Mar 2021 11:48:46 +0100 Subject: [PATCH 02/20] WIP: Tests --- frame/transaction-storage/src/lib.rs | 42 ++++++++++++++++--- primitives/io/src/lib.rs | 1 + .../src/overlayed_changes/mod.rs | 5 +++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 54a597b439a86..c5805092ab2db 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -120,7 +120,7 @@ pub mod pallet { #[pallet::call] impl Pallet { - #[pallet::weight(0)] + #[pallet::weight(1)] pub(super) fn store( origin: OriginFor, data: Vec, @@ -150,7 +150,7 @@ pub mod pallet { Ok(().into()) } - #[pallet::weight(0)] + #[pallet::weight(1)] pub(super) fn renew( origin: OriginFor, block: T::BlockNumber, @@ -172,7 +172,7 @@ pub mod pallet { Ok(().into()) } - #[pallet::weight(0)] + #[pallet::weight(1)] fn check_proof( origin: OriginFor, proof: Option, @@ -268,6 +268,27 @@ pub mod pallet { #[pallet::storage] pub(super) type StoragePeriod = StorageValue<_, T::BlockNumber>; + #[pallet::genesis_config] + pub struct GenesisConfig { + pub byte_fee: BalanceOf, + } + + #[cfg(feature = "std")] + impl Default for GenesisConfig { + fn default() -> Self { + Self { + byte_fee: 10u32.into(), + } + } + } + + #[pallet::genesis_build] + impl GenesisBuild for GenesisConfig { + fn build(&self) { + >::put(&self.byte_fee); + } + } + #[pallet::inherent] impl ProvideInherent for Pallet { type Call = Call; @@ -282,6 +303,10 @@ pub mod pallet { fn check_inherent(_call: &Self::Call, _data: &InherentData) -> result::Result<(), Self::Error> { Ok(()) } + + fn is_inherent(call: &Self::Call) -> bool { + matches!(call, Call::check_proof(_)) + } } impl Pallet { @@ -326,7 +351,7 @@ mod tests { { System: frame_system::{Pallet, Call, Config, Storage, Event}, Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Event}, + TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Config, Inherent, Event}, } ); @@ -384,11 +409,18 @@ mod tests { // We use default for brevity, but you can configure as desired if needed. frame_system: Default::default(), pallet_balances: Default::default(), + pallet_transaction_storage: pallet_transaction_storage::GenesisConfig::default(), }.build_storage().unwrap(); t.into() } #[test] - fn weights_work() { + fn stores_transaction() { + let mut ext = new_test_ext(); + let data = vec![1u8, 2u8, 3u8]; + ext.execute_with(|| { + assert_ok!(TransactionStorage::store(Origin::signed(1), data)); + }); + assert!(ext.overlayed_changes().transaction_index_ops().len() == 1); } } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 08902e9ebc9f3..0ef3b74d384ec 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1466,6 +1466,7 @@ pub type SubstrateHostFunctions = ( crate::trie::HostFunctions, offchain_index::HostFunctions, runtime_tasks::HostFunctions, + transaction_index::HostFunctions, ); #[cfg(test)] diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 2fe5947a0c798..c01d56ab919a0 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -520,6 +520,11 @@ impl OverlayedChanges { self.children.get(key).map(|(overlay, info)| (overlay.changes(), info)) } + /// Get an list of all index operations. + pub fn transaction_index_ops(&self) -> &[IndexOperation] { + &self.transaction_index_ops + } + /// Convert this instance with all changes into a [`StorageChanges`] instance. #[cfg(feature = "std")] pub fn into_storage_changes< From dd79ea55c5e7a5c609f5028e17da791634ad07ec Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 15 Apr 2021 14:00:23 +0300 Subject: [PATCH 03/20] Tests, benchmarks and docs --- Cargo.lock | 2 + bin/node/runtime/Cargo.toml | 2 + bin/node/runtime/src/lib.rs | 5 +- client/db/src/lib.rs | 7 +- frame/benchmarking/src/lib.rs | 29 ++- frame/transaction-storage/Cargo.toml | 11 +- frame/transaction-storage/README.md | 238 +----------------- frame/transaction-storage/src/benchmarking.rs | 141 +++++++++++ frame/transaction-storage/src/lib.rs | 203 +++++---------- frame/transaction-storage/src/mock.rs | 148 +++++++++++ frame/transaction-storage/src/tests.rs | 140 +++++++++++ frame/transaction-storage/src/weights.rs | 91 +++++++ primitives/storage-proof/src/lib.rs | 77 +++--- 13 files changed, 668 insertions(+), 426 deletions(-) create mode 100644 frame/transaction-storage/src/benchmarking.rs create mode 100644 frame/transaction-storage/src/mock.rs create mode 100644 frame/transaction-storage/src/tests.rs create mode 100644 frame/transaction-storage/src/weights.rs diff --git a/Cargo.lock b/Cargo.lock index ac63de633f69f..310cef0a8a912 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5607,9 +5607,11 @@ dependencies = [ name = "pallet-transaction-storage" version = "3.0.0" dependencies = [ + "frame-benchmarking", "frame-support", "frame-support-test", "frame-system", + "hex-literal", "pallet-balances", "parity-scale-codec", "serde", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index d53dc4013e77e..e57944674fcc4 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -153,6 +153,7 @@ std = [ "pallet-tips/std", "pallet-transaction-payment-rpc-runtime-api/std", "pallet-transaction-payment/std", + "pallet-transaction-storage/std", "pallet-treasury/std", "sp-transaction-pool/std", "pallet-utility/std", @@ -195,6 +196,7 @@ runtime-benchmarks = [ "pallet-staking/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-tips/runtime-benchmarks", + "pallet-transaction-storage/runtime-benchmarks", "pallet-treasury/runtime-benchmarks", "pallet-utility/runtime-benchmarks", "pallet-uniques/runtime-benchmarks", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 37b439cacea71..3094a5bb3ddb0 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1118,6 +1118,7 @@ impl pallet_transaction_storage::Config for Runtime { type Event = Event; type Currency = Balances; type Call = Call; + type WeightInfo = pallet_transaction_storage::weights::SubstrateWeight; } construct_runtime!( @@ -1164,11 +1165,8 @@ construct_runtime!( Mmr: pallet_mmr::{Pallet, Storage}, Lottery: pallet_lottery::{Pallet, Call, Storage, Event}, Gilt: pallet_gilt::{Pallet, Call, Storage, Event, Config}, -<<<<<<< HEAD Uniques: pallet_uniques::{Pallet, Call, Storage, Event}, -======= TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Event}, ->>>>>>> 589fb673e6 (Transaction storage runtime module) } ); @@ -1542,6 +1540,7 @@ impl_runtime_apis! { add_benchmark!(params, batches, frame_system, SystemBench::); add_benchmark!(params, batches, pallet_timestamp, Timestamp); add_benchmark!(params, batches, pallet_tips, Tips); + add_benchmark!(params, batches, pallet_transaction_storage, TransactionStorage); add_benchmark!(params, batches, pallet_treasury, Treasury); add_benchmark!(params, batches, pallet_uniques, Uniques); add_benchmark!(params, batches, pallet_utility, Utility); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index d3e451dc49af7..cda197ab0687a 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -3054,13 +3054,16 @@ pub(crate) mod tests { for i in 0 .. 10 { let mut index = Vec::new(); if i == 0 { - index.push(IndexOperation::Insert { extrinsic: 0, offset: 1 }); + index.push(IndexOperation::Insert { + extrinsic: 0, + hash: x1_hash.as_ref().to_vec(), + size: (x1.len() - 1) as u32, + }); } else if i < 5 { // keep renewing 1st index.push(IndexOperation::Renew { extrinsic: 0, hash: x1_hash.as_ref().to_vec(), - size: (x1.len() - 1) as u32, }); } // else stop renewing let hash = insert_block( diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 63f65db366651..8160bd5d1dd21 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -736,17 +736,20 @@ macro_rules! impl_benchmark { SelectedBenchmark as $crate::BenchmarkingSetup >::components(&selected_benchmark); + let mut progress = $crate::benchmarking::current_time(); // Default number of steps for a component. let mut prev_steps = 10; - let repeat_benchmark = | + let mut repeat_benchmark = | repeat: u32, c: &[($crate::BenchmarkParameter, u32)], results: &mut $crate::Vec<$crate::BenchmarkResults>, verify: bool, + step: u32, + num_steps: u32, | -> Result<(), &'static str> { // Run the benchmark `repeat` times. - for _ in 0..repeat { + for r in 0..repeat { // Set up the externalities environment for the setup we want to // benchmark. let closure_to_benchmark = < @@ -801,6 +804,20 @@ macro_rules! impl_benchmark { "Read/Write Count {:?}", read_write_count ); + let time = $crate::benchmarking::current_time(); + if time.saturating_sub(progress) > 5000000000 { + progress = $crate::benchmarking::current_time(); + $crate::log::info!( + target: "benchmark", + "Benchmarking {} {}/{}, run {}/{}", + extrinsic, + step, + num_steps, + r, + repeat, + ); + } + // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); $crate::storage_root(); @@ -829,9 +846,9 @@ macro_rules! impl_benchmark { if components.is_empty() { if verify { // If `--verify` is used, run the benchmark once to verify it would complete. - repeat_benchmark(1, Default::default(), &mut $crate::Vec::new(), true)?; + repeat_benchmark(1, Default::default(), &mut $crate::Vec::new(), true, 1, 1)?; } - repeat_benchmark(repeat, Default::default(), &mut results, false)?; + repeat_benchmark(repeat, Default::default(), &mut results, false, 1, 1)?; } else { // Select the component we will be benchmarking. Each component will be benchmarked. for (idx, (name, low, high)) in components.iter().enumerate() { @@ -869,9 +886,9 @@ macro_rules! impl_benchmark { if verify { // If `--verify` is used, run the benchmark once to verify it would complete. - repeat_benchmark(1, &c, &mut $crate::Vec::new(), true)?; + repeat_benchmark(1, &c, &mut $crate::Vec::new(), true, s, num_of_steps)?; } - repeat_benchmark(repeat, &c, &mut results, false)?; + repeat_benchmark(repeat, &c, &mut results, false, s, num_of_steps)?; } } } diff --git a/frame/transaction-storage/Cargo.toml b/frame/transaction-storage/Cargo.toml index 863408dc1ac52..3e11b60e38706 100644 --- a/frame/transaction-storage/Cargo.toml +++ b/frame/transaction-storage/Cargo.toml @@ -14,6 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } +hex-literal = { version = "0.3.1", optional = true } codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } frame-support = { version = "3.0.0", default-features = false, path = "../support" } frame-system = { version = "3.0.0", default-features = false, path = "../system" } @@ -23,13 +24,21 @@ sp-std = { version = "3.0.0", default-features = false, path = "../../primitives sp-io = { version = "3.0.0", default-features = false, path = "../../primitives/io" } sp-inherents = { version = "3.0.0", default-features = false, path = "../../primitives/inherents" } sp-storage-proof = { version = "3.0.0", default-features = false, path = "../../primitives/storage-proof" } +sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false, optional = true } +frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false } frame-support-test = { version = "3.0.0", path = "../support/test" } +sp-storage-proof = { version = "3.0.0", default-features = true, path = "../../primitives/storage-proof" } +sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false } [features] default = ["std"] +runtime-benchmarks = [ + "frame-benchmarking", + "sp-core", + "hex-literal", +] std = [ "serde", "codec/std", diff --git a/frame/transaction-storage/README.md b/frame/transaction-storage/README.md index 46a0d076a969a..4eaddae1786e7 100644 --- a/frame/transaction-storage/README.md +++ b/frame/transaction-storage/README.md @@ -1,238 +1,2 @@ - -# Example Pallet - -The Example: A simple example of a FRAME pallet demonstrating -concepts, APIs and structures common to most FRAME runtimes. - -Run `cargo doc --package pallet-example --open` to view this pallet's documentation. - -### Documentation Guidelines: - - - -
    -
  • Documentation comments (i.e. /// comment) - should - accompany pallet functions and be restricted to the pallet interface, - not the internals of the pallet implementation. Only state inputs, - outputs, and a brief description that mentions whether calling it - requires root, but without repeating the source code details. - Capitalize the first word of each documentation comment and end it with - a full stop. See - Generic example of annotating source code with documentation comments
    • -
  • Self-documenting code - Try to refactor code to be self-documenting.
    • -
  • Code comments - Supplement complex code with a brief explanation, not every line of code.
    • -
  • Identifiers - surround by backticks (i.e. INHERENT_IDENTIFIER, InherentType, - u64)
    • -
  • Usage scenarios - should be simple doctests. The compiler should ensure they stay valid.
    • -
  • Extended tutorials - should be moved to external files and refer to.
    • - -
  • Mandatory - include all of the sections/subsections where MUST is specified.
    • -
  • Optional - optionally include sections/subsections where CAN is specified.
    • -
- -### Documentation Template:
- -Copy and paste this template from frame/example/src/lib.rs into file -`frame//src/lib.rs` of your own custom pallet and complete it. -

-// Add heading with custom pallet name
-
-\#  Pallet
-
-// Add simple description
-
-// Include the following links that shows what trait needs to be implemented to use the pallet
-// and the supported dispatchables that are documented in the Call enum.
-
-- \[`::Trait`](https://docs.rs/pallet-example/latest/pallet_example/trait.Trait.html)
-- \[`Call`](https://docs.rs/pallet-example/latest/pallet_example/enum.Call.html)
-- \[`Module`](https://docs.rs/pallet-example/latest/pallet_example/struct.Module.html)
-
-\## Overview
-
-
-// Short description of pallet's purpose.
-// Links to Traits that should be implemented.
-// What this pallet is for.
-// What functionality the pallet provides.
-// When to use the pallet (use case examples).
-// How it is used.
-// Inputs it uses and the source of each input.
-// Outputs it produces.
-
-
-
-
-\## Terminology
-
-// Add terminology used in the custom pallet. Include concepts, storage items, or actions that you think
-// deserve to be noted to give context to the rest of the documentation or pallet usage. The author needs to
-// use some judgment about what is included. We don't want a list of every storage item nor types - the user
-// can go to the code for that. For example, "transfer fee" is obvious and should not be included, but
-// "free balance" and "reserved balance" should be noted to give context to the pallet.
-// Please do not link to outside resources. The reference docs should be the ultimate source of truth.
-
-
-
-\## Goals
-
-// Add goals that the custom pallet is designed to achieve.
-
-
-
-\### Scenarios
-
-
-
-\#### 
-
-// Describe requirements prior to interacting with the custom pallet.
-// Describe the process of interacting with the custom pallet for this scenario and public API functions used.
-
-\## Interface
-
-\### Supported Origins
-
-// What origins are used and supported in this pallet (root, signed, none)
-// i.e. root when \`ensure_root\` used
-// i.e. none when \`ensure_none\` used
-// i.e. signed when \`ensure_signed\` used
-
-\`inherent\` 
-
-
-
-
-\### Types
-
-// Type aliases. Include any associated types and where the user would typically define them.
-
-\`ExampleType\` 
-
-
-
-// Reference documentation of aspects such as `storageItems` and `dispatchable` functions should only be
-// included in the https://docs.rs Rustdocs for Substrate and not repeated in the README file.
-
-\### Dispatchable Functions
-
-
-
-// A brief description of dispatchable functions and a link to the rustdoc with their actual documentation.
-
-// MUST have link to Call enum
-// MUST have origin information included in function doc
-// CAN have more info up to the user
-
-\### Public Functions
-
-
-
-// A link to the rustdoc and any notes about usage in the pallet, not for specific functions.
-// For example, in the Balances Pallet: "Note that when using the publicly exposed functions,
-// you (the runtime developer) are responsible for implementing any necessary checks
-// (e.g. that the sender is the signer) before calling a function that will affect storage."
-
-
-
-// It is up to the writer of the respective pallet (with respect to how much information to provide).
-
-\#### Public Inspection functions - Immutable (getters)
-
-// Insert a subheading for each getter function signature
-
-\##### \`example_getter_name()\`
-
-// What it returns
-// Why, when, and how often to call it
-// When it could panic or error
-// When safety issues to consider
-
-\#### Public Mutable functions (changing state)
-
-// Insert a subheading for each setter function signature
-
-\##### \`example_setter_name(origin, parameter_name: T::ExampleType)\`
-
-// What state it changes
-// Why, when, and how often to call it
-// When it could panic or error
-// When safety issues to consider
-// What parameter values are valid and why
-
-\### Storage Items
-
-// Explain any storage items included in this pallet
-
-\### Digest Items
-
-// Explain any digest items included in this pallet
-
-\### Inherent Data
-
-// Explain what inherent data (if any) is defined in the pallet and any other related types
-
-\### Events:
-
-// Insert events for this pallet if any
-
-\### Errors:
-
-// Explain what generates errors
-
-\## Usage
-
-// Insert 2-3 examples of usage and code snippets that show how to
-// use  Pallet in a custom pallet.
-
-\### Prerequisites
-
-// Show how to include necessary imports for  and derive
-// your pallet configuration trait with the `INSERT_CUSTOM_PALLET_NAME` trait.
-
-\```rust
-use ;
-
-pub trait Config: ::Config { }
-\```
-
-\### Simple Code Snippet
-
-// Show a simple example (e.g. how to query a public getter function of )
-
-\### Example from FRAME
-
-// Show a usage example in an actual runtime
-
-// See:
-// - Substrate TCR https://github.com/parity-samples/substrate-tcr
-// - Substrate Kitties https://shawntabrizi.github.io/substrate-collectables-workshop/#/
-
-\## Genesis Config
-
-
-
-\## Dependencies
-
-// Dependencies on other FRAME pallets and the genesis config should be mentioned,
-// but not the Rust Standard Library.
-// Genesis configuration modifications that may be made to incorporate this pallet
-// Interaction with other pallets
-
-
-
-\## Related Pallets
-
-// Interaction with other pallets in the form of a bullet point list
-
-\## References
-
-
-
-// Links to reference material, if applicable. For example, Phragmen, W3F research, etc.
-// that the implementation is based on.
-

- -License: Unlicense +License: Apache-2.0 diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs new file mode 100644 index 0000000000000..490a81632ce9d --- /dev/null +++ b/frame/transaction-storage/src/benchmarking.rs @@ -0,0 +1,141 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Benchmarks for transction-storage Pallet + +#![cfg(feature = "runtime-benchmarks")] + +use sp_std::*; +use super::*; +use sp_runtime::traits::{Zero, One, Bounded}; +use frame_system::{RawOrigin, Pallet as System, EventRecord}; +use frame_benchmarking::{benchmarks, whitelisted_caller, impl_benchmark_test_suite}; +use frame_support::{traits::{Currency}}; + +use crate::Pallet as TransactionStorage; +use crate::mock::{run_to_block, setup}; + +const MAX_EXTRINSICS: u32 = 100; + +const PROOF: &[u8] = &hex_literal::hex!(" + 0104000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000014cd0780ffff80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe8 + 7d12a3662c4c0080e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb + 13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2 + f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f + 1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f + 3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a47 + 8e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cf + f93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e31 + 6a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f + 53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c8 + 0e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4cbd05807777809a5d7a720ce5f9d9a012 + fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf + 3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9 + a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062d + c1cf3ac289390ae4c00809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a + 3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c + 4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bc + bf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce + 186c4ddc53f118e0ddd4decd8cccd0780ffff8081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb0 + 3bdb31008081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253 + 515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa139 + 8e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5 + f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3a + a1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2b + a8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f32 + 2d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa + 9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f0 + 2f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b82 + 5bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb31cd0780ffff80b4f23ac50c8e67d9b280f2b31a + 5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd1885 + 44c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2 + b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd + 188544c5f9b0080b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9 + b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84 + d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e + 67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977aca + ac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac5 + 0c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b89297 + 7acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b104401 + 0000 +"); + +type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +fn assert_last_event(generic_event: ::Event) { + let events = System::::events(); + let system_event: ::Event = generic_event.into(); + let EventRecord { event, .. } = &events[events.len() - 1]; + assert_eq!(event, &system_event); +} + +benchmarks! { + store { + let l in 1 .. MAX_DATA_SIZE; + setup::()?; + let caller: T::AccountId = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + }: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize]) + verify { + assert!(TransactionRoots::::get(T::BlockNumber::one(), 0u32).is_some()); + assert_last_event::(Event::Stored(0).into()); + } + + renew { + setup::()?; + let caller: T::AccountId = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8; MAX_DATA_SIZE as usize] + )?; + }: _(RawOrigin::Signed(caller.clone()), T::BlockNumber::zero(), 0) + verify { + assert_last_event::(Event::Renewed(1).into()); + } + + check_proof_max { + setup::()?; + run_to_block::(1); + let caller: T::AccountId = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + for _ in 0 .. MAX_EXTRINSICS { + TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8; MAX_DATA_SIZE as usize] + )?; + } + run_to_block::(11); + let random_hash = [0u8]; + let mut encoded_proof = PROOF; + let proof = sp_storage_proof::StorageProof::decode(&mut encoded_proof).unwrap(); + }: check_proof(RawOrigin::None, Some(proof)) + verify { + assert_last_event::(Event::ProofChecked.into()); + } +} + +impl_benchmark_test_suite!( + TransactionStorage, + crate::mock::runtime::new_test_ext(), + crate::mock::runtime::Test, +); diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index c5805092ab2db..fd98870fc0914 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -17,10 +17,16 @@ //! Transaction storage pallet. Indexes transactions and manages storage proofs. - // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(any(test, feature = "runtime-benchmarks"))] +mod mock; +#[cfg(test)] +mod tests; +mod benchmarking; +pub mod weights; + use frame_support::{ traits::{ReservableCurrency, Currency}, dispatch::{Dispatchable, GetDispatchInfo}, @@ -28,50 +34,44 @@ use frame_support::{ use sp_std::prelude::*; use sp_std::{result}; use codec::{Encode, Decode}; -use sp_runtime::{ - traits::{ - Saturating, BlakeTwo256, Hash, Zero, - }, -}; -use sp_storage_proof::StorageProof; +use sp_runtime::traits::{Saturating, BlakeTwo256, Hash, Zero}; +use sp_storage_proof::{StorageProof, DEFAULT_STORAGE_PERIOD}; /// A type alias for the balance type from this pallet's point of view. type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; // Re-export pallet items so that they can be accessed from the crate namespace. pub use pallet::*; +pub use weights::WeightInfo; const CHUNK_SIZE: usize = sp_storage_proof::CHUNK_SIZE; +/// Maximum bytes that can be storead in one transaction. +// Increasing it further also requires raising the allocator limit. +pub const MAX_DATA_SIZE: u32 = 8 * 1024 * 1024; -#[derive(Encode, Decode, sp_runtime::RuntimeDebug)] +#[derive(Encode, Decode, sp_runtime::RuntimeDebug, PartialEq, Eq)] pub struct TransactionInfo { chunk_root: ::Output, content_hash: ::Output, size: u32, } -// Definition of the pallet logic, to be aggregated at runtime definition through -// `construct_runtime`. #[frame_support::pallet] pub mod pallet { - // Import various types used to declare pallet in scope. use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use super::*; - /// Our pallet's configuration trait. All our types and constants go in here. If the - /// pallet is dependent on specific other pallets, then their configuration traits - /// should be added to our implied traits list. - /// - /// `frame_system::Config` should always be included. #[pallet::config] - pub trait Config: pallet_balances::Config + frame_system::Config { + pub trait Config: frame_system::Config { /// The overarching event type. type Event: From> + IsType<::Event>; /// A dispatchable call. type Call: Parameter + Dispatchable + GetDispatchInfo + From>; /// The currency trait. type Currency: ReservableCurrency; + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; } #[pallet::error] @@ -101,15 +101,16 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(_n: T::BlockNumber) -> Weight { + // This will be cleared in in `on_finalize` >::put(0); 0 } fn on_finalize(n: T::BlockNumber) { >::insert(n, >::get().unwrap_or(0)); - // Drop obsolete roots >::kill(); - let period = >::get().unwrap(); + // Drop obsolete roots + let period = >::get().unwrap_or(DEFAULT_STORAGE_PERIOD.into()); let obsolete = n.saturating_sub(period); if obsolete > Zero::zero() { >::remove_prefix(obsolete); @@ -120,7 +121,13 @@ pub mod pallet { #[pallet::call] impl Pallet { - #[pallet::weight(1)] + /// Index and store data on chain. Minimum data size is 1 bytes, maxmum is `MAX_DATA_SIZE`. + /// Data will be removed after `STORAGE_PERIOD` blocks, unless `renew` is called. + /// # + /// - n*log(n) of data size, as all data is pushed to an in-memory trie. + /// Additionally contains a DB write. + /// # + #[pallet::weight(T::WeightInfo::store(data.len() as u32))] pub(super) fn store( origin: OriginFor, data: Vec, @@ -143,14 +150,20 @@ pub mod pallet { size: data.len() as u32, content_hash: content_hash.into(), }); + Self::deposit_event(Event::Stored(counter)); counter += 1; >::put(counter); - - Self::deposit_event(Event::Stored); Ok(().into()) } - #[pallet::weight(1)] + /// Renew previously stored data. Parameters are the block number that contains + /// previous `store` or `renew` call and transaction index within that block. + /// Transaction index is emitted in the `Stored` or `Renewed` event. + /// Applies same fees as `store`. + /// # + /// - Constant with a single DB read. + /// # + #[pallet::weight(T::WeightInfo::renew())] pub(super) fn renew( origin: OriginFor, block: T::BlockNumber, @@ -165,21 +178,27 @@ pub mod pallet { let mut counter = >::get().unwrap_or(0); let block_num = >::block_number(); >::insert(block_num, counter, info); + Self::deposit_event(Event::Renewed(counter)); counter += 1; >::put(counter); - - Self::deposit_event(Event::Renewed); Ok(().into()) } - #[pallet::weight(1)] - fn check_proof( + /// Check storage proof for block number `block_number() - StoragePeriod`. + /// If such block does not exist the proof is expected to be `None`. + /// # + /// - Linear w.r.t the number of indexed transactions in the proved block for random probing. + /// There's a DB read for each transaction. + /// Here we assume a maximum of 100 probed transactions. + /// # + #[pallet::weight(T::WeightInfo::check_proof_max())] + pub(super) fn check_proof( origin: OriginFor, proof: Option, ) -> DispatchResultWithPostInfo { ensure_none(origin)?; let number = >::block_number(); - let period = >::get().unwrap(); + let period = >::get().unwrap_or(DEFAULT_STORAGE_PERIOD.into()); let target_number = number.saturating_sub(period); if target_number.is_zero() { ensure!(proof.is_none(), Error::::UnexpectedProof); @@ -189,7 +208,8 @@ pub mod pallet { let mut total_chunks = 0; for t in 0 .. transaction_count { match >::get(target_number, t) { - Some(info) => total_chunks += ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32, + Some(info) => total_chunks += + ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32, None => break, } } @@ -205,10 +225,11 @@ pub mod pallet { let (info, chunk_index) = loop { match >::get(target_number, t) { Some(info) => { - total_chunks += ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32; - if total_chunks >= selected_chunk_index { - break (info, total_chunks - selected_chunk_index) + let chunks = ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32; + if total_chunks + chunks >= selected_chunk_index { + break (info, selected_chunk_index - total_chunks) } + total_chunks += chunks; }, None => Err(Error::::MissingStateData)?, } @@ -219,11 +240,12 @@ pub mod pallet { sp_io::trie::blake2_256_verity_proof( info.chunk_root, &proof.proof, - &(chunk_index as u32).to_be_bytes(), + &sp_storage_proof::encode_index(chunk_index), &proof.chunk, ), Error::::InvalidProof ); + Self::deposit_event(Event::ProofChecked); Ok(().into()) } } @@ -231,8 +253,12 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - Stored, - Renewed, + /// Stored data under specified index. + Stored(u32), + /// Renewed data under specified index. + Renewed(u32), + /// Storage proof was successfully checked. + ProofChecked, } /// Collection of extrinsic storage roots for the current block. @@ -260,11 +286,14 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn byte_fee)] + /// Storage fee per byte. pub(super) type ByteFee = StorageValue<_, BalanceOf>; #[pallet::storage] pub(super) type Counter = StorageValue<_, u32>; + /// Storage period for data in blocks. Should match `sp_storage_proof::DEFAULT_STORAGE_PERIOD` + /// for block authoring. #[pallet::storage] pub(super) type StoragePeriod = StorageValue<_, T::BlockNumber>; @@ -320,107 +349,3 @@ pub mod pallet { } } } - -#[cfg(test)] -mod tests { - use super::*; - - use frame_support::{ - assert_ok, parameter_types, - weights::{DispatchInfo, GetDispatchInfo}, traits::{OnInitialize, OnFinalize} - }; - use sp_core::H256; - // The testing primitives are very useful for avoiding having to work with signatures - // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. - use sp_runtime::{ - testing::Header, BuildStorage, - traits::{BlakeTwo256, IdentityLookup}, - }; - // Reexport crate as its pallet name for construct_runtime. - use crate as pallet_transaction_storage; - - type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; - type Block = frame_system::mocking::MockBlock; - - // For testing the pallet, we construct a mock runtime. - frame_support::construct_runtime!( - pub enum Test where - Block = Block, - NodeBlock = Block, - UncheckedExtrinsic = UncheckedExtrinsic, - { - System: frame_system::{Pallet, Call, Config, Storage, Event}, - Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Config, Inherent, Event}, - } - ); - - parameter_types! { - pub const BlockHashCount: u64 = 250; - pub BlockWeights: frame_system::limits::BlockWeights = - frame_system::limits::BlockWeights::simple_max(1024); - } - impl frame_system::Config for Test { - type BaseCallFilter = (); - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); - type Origin = Origin; - type Index = u64; - type BlockNumber = u64; - type Hash = H256; - type Call = Call; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Header = Header; - type Event = Event; - type BlockHashCount = BlockHashCount; - type Version = (); - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - } - parameter_types! { - pub const ExistentialDeposit: u64 = 1; - } - impl pallet_balances::Config for Test { - type MaxLocks = (); - type Balance = u64; - type DustRemoval = (); - type Event = Event; - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type WeightInfo = (); - } - impl Config for Test { - type Event = Event; - type Call = Call; - type Currency = Balances; - } - - // This function basically just builds a genesis storage key/value store according to - // our desired mockup. - pub fn new_test_ext() -> sp_io::TestExternalities { - let t = GenesisConfig { - // We use default for brevity, but you can configure as desired if needed. - frame_system: Default::default(), - pallet_balances: Default::default(), - pallet_transaction_storage: pallet_transaction_storage::GenesisConfig::default(), - }.build_storage().unwrap(); - t.into() - } - - #[test] - fn stores_transaction() { - let mut ext = new_test_ext(); - let data = vec![1u8, 2u8, 3u8]; - ext.execute_with(|| { - assert_ok!(TransactionStorage::store(Origin::signed(1), data)); - }); - assert!(ext.overlayed_changes().transaction_index_ops().len() == 1); - } -} diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs new file mode 100644 index 0000000000000..c91e9afcfe619 --- /dev/null +++ b/frame/transaction-storage/src/mock.rs @@ -0,0 +1,148 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test environment for transaction-storage pallet. + +use crate as pallet_transaction_storage; +use crate::Config; + +use frame_support::{ + traits::{OnInitialize, OnFinalize, Currency}, +}; +use sp_runtime::traits::One; + +type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +#[cfg(test)] +pub mod runtime { + use super::*; + use sp_core::H256; + use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::Header, BuildStorage}; + use frame_support::parameter_types; + type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; + pub type Block = frame_system::mocking::MockBlock; + + + // Configure a mock runtime to test the pallet. + frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Config, Storage, Event}, + TransactionStorage: pallet_transaction_storage::{ + Pallet, Call, Storage, Config, Inherent, Event + }, + } + ); + + parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const SS58Prefix: u8 = 42; + } + + impl frame_system::Config for Test { + type BaseCallFilter = (); + type BlockWeights = (); + type BlockLength = (); + type Origin = Origin; + type Call = Call; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = SS58Prefix; + type OnSetCode = (); + } + + parameter_types! { + pub const ExistentialDeposit: u64 = 1; + } + + impl pallet_balances::Config for Test { + type Balance = u64; + type DustRemoval = (); + type Event = Event; + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type WeightInfo = (); + type MaxLocks = (); + } + + parameter_types! { + pub IgnoredIssuance: u64 = Balances::total_balance(&0); // Account zero is ignored. + pub const QueueCount: u32 = 3; + pub const MaxQueueLen: u32 = 3; + pub const FifoQueueLen: u32 = 1; + pub const Period: u64 = 3; + pub const MinFreeze: u64 = 2; + pub const IntakePeriod: u64 = 2; + pub const MaxIntakeBids: u32 = 2; + } + + impl pallet_transaction_storage::Config for Test { + type Event = Event; + type Call = Call; + type Currency = Balances; + type WeightInfo = (); + } + + pub fn new_test_ext() -> sp_io::TestExternalities { + let t = GenesisConfig { + frame_system: Default::default(), + pallet_balances: pallet_balances::GenesisConfig:: { + balances: vec![(1, 1000000000), (2, 100), (3, 100), (4, 100)] + }, + pallet_transaction_storage: pallet_transaction_storage::GenesisConfig::default(), + }.build_storage().unwrap(); + t.into() + } + +} + +pub fn run_to_block(n: u32) { + while frame_system::Pallet::::block_number() < (n as u32).into() { + pallet_transaction_storage::Pallet::::on_finalize(frame_system::Pallet::::block_number()); + frame_system::Pallet::::on_finalize(frame_system::Pallet::::block_number()); + frame_system::Pallet::::set_block_number(frame_system::Pallet::::block_number() + One::one()); + frame_system::Pallet::::on_initialize(frame_system::Pallet::::block_number()); + pallet_transaction_storage::Pallet::::on_initialize(frame_system::Pallet::::block_number()); + } +} + +pub fn setup() -> Result<(), &'static str> { + let fee = BalanceOf::::from(2u32); + let storage_period = 10u32.into(); + pallet_transaction_storage::ByteFee::::set(Some(fee)); + pallet_transaction_storage::StoragePeriod::::set(Some(storage_period)); + Ok(()) +} + diff --git a/frame/transaction-storage/src/tests.rs b/frame/transaction-storage/src/tests.rs new file mode 100644 index 0000000000000..903c78e845fef --- /dev/null +++ b/frame/transaction-storage/src/tests.rs @@ -0,0 +1,140 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tests for transction-stroage pallet. + +use super::*; +use crate::mock::*; +use crate::mock::runtime::*; +use super::Pallet as TransactionStorage; +use frame_support::{assert_ok, assert_noop}; +use frame_system::RawOrigin; + +#[test] +fn discards_data() { + new_test_ext().execute_with(|| { + setup::().unwrap(); + run_to_block::(1); + let caller = 1; + assert_ok!(TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8; 2000 as usize] + )); + assert_ok!(TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8; 2000 as usize] + )); + run_to_block::(11); + assert!(TransactionRoots::::get(1, 0).is_some()); + assert!(TransactionRoots::::get(1, 1).is_some()); + assert!(TransactionRoots::::get(1, 2).is_none()); + assert_eq!(TransactionCount::::get(1), 2); + run_to_block::(12); + assert!(TransactionRoots::::get(1, 0).is_none()); + assert!(TransactionRoots::::get(1, 1).is_none()); + assert_eq!(TransactionCount::::get(1), 0); + }); +} + +#[test] +fn burns_fee() { + new_test_ext().execute_with(|| { + setup::().unwrap(); + run_to_block::(1); + let caller = 1; + assert_noop!(TransactionStorage::::store( + RawOrigin::Signed(5).into(), + vec![0u8; 2000 as usize] + ), + Error::::InsufficientFunds, + ); + assert_ok!(TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8; 2000 as usize] + )); + assert_eq!(Balances::free_balance(1), 1_000_000_000 - 2000 * 2); + }); +} + +#[test] +fn checks_proof() { + new_test_ext().execute_with(|| { + setup::().unwrap(); + run_to_block::(1); + let caller = 1; + assert_ok!(TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8; MAX_DATA_SIZE as usize] + )); + run_to_block::(10); + let parent_hash = >::parent_hash(); + let proof = sp_storage_proof::registration::build_proof( + parent_hash.as_ref(), + vec![vec![0u8; MAX_DATA_SIZE as usize]] + ).unwrap(); + assert_noop!(TransactionStorage::::check_proof( + Origin::none(), + Some(proof), + ), + Error::::UnexpectedProof, + ); + run_to_block::(11); + let parent_hash = >::parent_hash(); + let proof = sp_storage_proof::registration::build_proof( + parent_hash.as_ref(), + vec![vec![0u8; MAX_DATA_SIZE as usize]] + ).unwrap(); + assert_ok!(TransactionStorage::::check_proof(Origin::none(), Some(proof))); + let invalid_proof = sp_storage_proof::registration::build_proof( + parent_hash.as_ref(), + vec![vec![0u8; 1000]] + ).unwrap(); + assert_noop!(TransactionStorage::::check_proof( + Origin::none(), + Some(invalid_proof), + ), + Error::::InvalidProof, + ); + }); +} + +#[test] +fn renews_data() { + new_test_ext().execute_with(|| { + setup::().unwrap(); + run_to_block::(1); + let caller = 1; + assert_ok!(TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8; 2000 as usize] + )); + let info = TransactionRoots::::get(1, 0).unwrap(); + run_to_block::(6); + assert_ok!(TransactionStorage::::renew( + RawOrigin::Signed(caller.clone()).into(), + 1, // block + 0, // transaction + )); + assert_eq!(Balances::free_balance(1), 1_000_000_000 - 4000 * 2); + run_to_block::(16); + assert!(TransactionRoots::::get(1, 0).is_none()); + assert_eq!(TransactionRoots::::get(6, 0), Some(info)); + run_to_block::(17); + assert!(TransactionRoots::::get(6, 0).is_none()); + }); +} + diff --git a/frame/transaction-storage/src/weights.rs b/frame/transaction-storage/src/weights.rs new file mode 100644 index 0000000000000..b20d22c30e5c3 --- /dev/null +++ b/frame/transaction-storage/src/weights.rs @@ -0,0 +1,91 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_transaction_storage +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 +//! DATE: 2021-04-15, STEPS: `[10, ]`, REPEAT: 10, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 + +// Executed Command: +// target/release/substrate +// benchmark +// --chain=dev +// --steps=10 +// --repeat=10 +// --pallet=pallet_transaction-storage +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/transaction-storage/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + + +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_transaction_storage. +pub trait WeightInfo { + fn store(l: u32, ) -> Weight; + fn renew() -> Weight; + fn check_proof_max() -> Weight; +} + +/// Weights for pallet_transaction_storage using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + fn store(l: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((12_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + fn renew() -> Weight { + (120_000_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + fn check_proof_max() -> Weight { + (1_482_000_000 as Weight) + .saturating_add(T::DbWeight::get().reads(103 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + fn store(l: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((12_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + fn renew() -> Weight { + (120_000_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + fn check_proof_max() -> Weight { + (1_482_000_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(103 as Weight)) + } +} diff --git a/primitives/storage-proof/src/lib.rs b/primitives/storage-proof/src/lib.rs index 808544d4fb641..afd7a1f826cdf 100644 --- a/primitives/storage-proof/src/lib.rs +++ b/primitives/storage-proof/src/lib.rs @@ -15,7 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Storge Proof Primitives +//! Storge proof primitives. Constains types and basic code to extract storage +//! proofs for indexed transactions. #![cfg_attr(not(feature = "std"), no_std)] @@ -26,7 +27,9 @@ use sp_inherents::{Error, InherentIdentifier, InherentData, IsFatalError}; /// The identifier for the proof inherent. pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"tx_proof"; -pub const STORAGE_PERIOD: u32 = 10; +/// Storage period for data. +pub const DEFAULT_STORAGE_PERIOD: u32 = 100800; +/// Proof trie value size. pub const CHUNK_SIZE: usize = 256; /// Errors that can occur while checking the storage proof. @@ -98,7 +101,7 @@ where F: Fn() -> Result, Error> } } - +/// An utility function to extract chunk index from the source of randomness. pub fn random_chunk(random_hash: &[u8], total_chunks: u32) -> u32 { let mut buf = [0u8; 8]; buf.copy_from_slice(&random_hash[0..8]); @@ -106,6 +109,11 @@ pub fn random_chunk(random_hash: &[u8], total_chunks: u32) -> u32 { (random_u64 % total_chunks as u64) as u32 } +/// An utility function to enocde transaction index as trie key. +pub fn encode_index(input: u32) -> Vec { + codec::Encode::encode(&codec::Compact(input)) +} + #[cfg(feature = "std")] pub mod registration { use sp_consensus::SelectChain; @@ -114,13 +122,11 @@ pub mod registration { use sc_client_api::{HeaderBackend, BlockBackend}; use sp_runtime::{traits::{Block as BlockT, Header, Saturating, Zero}, generic::BlockId}; use std::sync::Arc; - use sp_trie::{TrieConfiguration, TrieMut}; - use crate::{StorageProof, CHUNK_SIZE}; - use std::iter::once; + use sp_trie::TrieMut; + use super::*; - type TrieLayout = sp_trie::Layout::; type Hasher = sp_core::Blake2Hasher; - + type TrieLayout = sp_trie::Layout::; /// Register uncles inherent data provider, if not registered already. pub fn register_storage_proof_inherent_data_provider( @@ -132,9 +138,9 @@ pub mod registration { C: BlockBackend + HeaderBackend + Send + Sync + 'static, SC: SelectChain + 'static, { - if !inherent_data_providers.has_provider(&crate::INHERENT_IDENTIFIER) { + if !inherent_data_providers.has_provider(&INHERENT_IDENTIFIER) { inherent_data_providers - .register_provider(crate::InherentDataProvider::new(move || { + .register_provider(InherentDataProvider::new(move || { { let chain_head = match select_chain.best_chain() { Ok(x) => x, @@ -144,7 +150,7 @@ pub mod registration { } }; - let number = chain_head.number().saturating_sub(crate::STORAGE_PERIOD.into()); + let number = chain_head.number().saturating_sub(DEFAULT_STORAGE_PERIOD.into()); if number.is_zero() { // Too early to collect proofs. return Ok(None); @@ -152,7 +158,7 @@ pub mod registration { match client.block_indexed_body(&BlockId::number(number)) { Ok(Some(transactions)) => { - Ok(Some(build_proof::(chain_head.parent_hash().as_ref(), transactions)?)) + Ok(Some(build_proof(chain_head.parent_hash().as_ref(), transactions)?)) }, Ok(None) => { // Nothing was indexed in that block. @@ -170,16 +176,15 @@ pub mod registration { Ok(()) } - fn build_proof(random_hash: &[u8], transactions: Vec>) + /// Build a proof for a given source of randomness and indexed transactions. + pub fn build_proof(random_hash: &[u8], transactions: Vec>) -> Result { - let mut block_root = sp_trie::empty_trie_root::(); let mut db = sp_trie::MemoryDB::::default(); let mut target_chunk = None; let mut target_root = Default::default(); let mut target_chunk_key = Default::default(); - let mut target_block_key = Default::default(); let mut chunk_proof = Default::default(); let total_chunks: u64 = transactions.iter().map(|t| ((t.len() + CHUNK_SIZE - 1) / CHUNK_SIZE) as u64).sum(); @@ -187,16 +192,15 @@ pub mod registration { buf.copy_from_slice(&random_hash[0..8]); let random_u64 = u64::from_be_bytes(buf); let target_chunk_index = random_u64 % total_chunks; - //Generate tries for each - + //Generate tries for each transaction. let mut chunk_index = 0; - for (ti, transaction) in transactions.into_iter().enumerate() { + for transaction in transactions { let mut transaction_root = sp_trie::empty_trie_root::(); { let mut trie = sp_trie::TrieDBMut::::new(&mut db, &mut transaction_root); - let chunks = transaction.chunks(crate::CHUNK_SIZE).map(|c| c.to_vec()); + let chunks = transaction.chunks(CHUNK_SIZE).map(|c| c.to_vec()); for (index, chunk) in chunks.enumerate() { - let index = TrieLayout::encode_index(index as u32); + let index = encode_index(index as u32); trie.insert(&index, &chunk) .map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; if chunk_index == target_chunk_index { @@ -207,36 +211,33 @@ pub mod registration { } trie.commit(); } - let transaction_key = TrieLayout::encode_index(ti as u32); if target_chunk.is_some() && target_root == Default::default() { target_root = transaction_root.clone(); - target_block_key = transaction_key.clone(); chunk_proof = sp_trie::generate_trie_proof::( &db, transaction_root.clone(), &[target_chunk_key.clone()] ).map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; } - { - let mut block_trie = sp_trie::TrieDBMut::::new(&mut db, &mut block_root); - block_trie.insert(&transaction_key, transaction_root.as_ref()) - .map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; - block_trie.commit(); - } }; - let block_proof = sp_trie::generate_trie_proof::( - &db, - block_root, - &[target_block_key] - ).map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; - - let proof = sp_trie::StorageProof::merge( - once(sp_trie::StorageProof::new(chunk_proof)).chain( - once(sp_trie::StorageProof::new(block_proof)))); Ok(StorageProof { - proof: proof.into_nodes(), + proof: chunk_proof, chunk: target_chunk.unwrap(), }) } + + #[test] + fn build_proof_check() { + use std::str::FromStr; + let random = [0u8; 32]; + let proof = build_proof(&random, vec![vec![42]]).unwrap(); + let root = sp_core::H256::from_str("0xff8611a4d212fc161dae19dd57f0f1ba9309f45d6207da13f2d3eab4c6839e91").unwrap(); + sp_trie::verify_trie_proof::( + &root, + &proof.proof, + &[(encode_index(0), Some(proof.chunk))], + ).unwrap(); + } } + From 57e1d93f8619f62e334c3e8a319884cb3740dafb Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 15 Apr 2021 17:29:33 +0300 Subject: [PATCH 04/20] Made check_proof mandatory --- bin/node/bench/src/import.rs | 8 ++++---- frame/transaction-storage/src/lib.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index b4fee58dac025..1da284217884a 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -140,7 +140,7 @@ impl core::Benchmark for ImportBenchmark { match self.block_type { BlockType::RandomTransfersKeepAlive => { // should be 5 per signed extrinsic + 1 per unsigned - // we have 1 unsigned and the rest are signed in the block + // we have 2 unsigned and the rest are signed in the block // those 5 events per signed are: // - new account (RawEvent::NewAccount) as we always transfer fund to non-existant account // - endowed (RawEvent::Endowed) for this new account @@ -149,7 +149,7 @@ impl core::Benchmark for ImportBenchmark { // - extrinsic success assert_eq!( node_runtime::System::events().len(), - (self.block.extrinsics.len() - 1) * 5 + 1, + (self.block.extrinsics.len() - 2) * 5 + 2, ); }, BlockType::Noop => { @@ -157,11 +157,11 @@ impl core::Benchmark for ImportBenchmark { node_runtime::System::events().len(), // should be 2 per signed extrinsic + 1 per unsigned - // we have 1 unsigned and the rest are signed in the block + // we have 2 unsigned and the rest are signed in the block // those 2 events per signed are: // - deposit event for charging transaction fee // - extrinsic success - (self.block.extrinsics.len() - 1) * 2 + 1, + (self.block.extrinsics.len() - 2) * 2 + 2, ); }, _ => {}, diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index fd98870fc0914..c6ba9e7408941 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -191,7 +191,7 @@ pub mod pallet { /// There's a DB read for each transaction. /// Here we assume a maximum of 100 probed transactions. /// # - #[pallet::weight(T::WeightInfo::check_proof_max())] + #[pallet::weight((T::WeightInfo::check_proof_max(), DispatchClass::Mandatory))] pub(super) fn check_proof( origin: OriginFor, proof: Option, From d0af6a069024ef06a8cbb06f96bf69ce9eca0cbf Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 16 Apr 2021 09:32:44 +0300 Subject: [PATCH 05/20] Typo --- frame/transaction-storage/src/lib.rs | 2 +- primitives/io/src/lib.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index c6ba9e7408941..4490eb9c0a292 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -237,7 +237,7 @@ pub mod pallet { }; ensure!( - sp_io::trie::blake2_256_verity_proof( + sp_io::trie::blake2_256_verify_proof( info.chunk_root, &proof.proof, &sp_storage_proof::encode_index(chunk_index), diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 0ef3b74d384ec..f0fcc4f1b0672 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -431,7 +431,7 @@ pub trait Trie { } /// Verify trie proof - fn blake2_256_verity_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { + fn blake2_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { sp_trie::verify_trie_proof::, _, _, _>( &root, proof, @@ -440,7 +440,7 @@ pub trait Trie { } /// Verify trie proof - fn keccak_256_verity_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { + fn keccak_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { sp_trie::verify_trie_proof::, _, _, _>( &root, proof, From 419bc5d74d0d505b2519e215209b235f2beb3688 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 16 Apr 2021 09:54:57 +0300 Subject: [PATCH 06/20] Renamed a crate --- Cargo.lock | 32 +++++++++---------- Cargo.toml | 2 +- frame/transaction-storage/Cargo.toml | 4 +-- frame/transaction-storage/src/benchmarking.rs | 3 +- frame/transaction-storage/src/lib.rs | 19 ++++++----- frame/transaction-storage/src/tests.rs | 7 ++-- .../Cargo.toml | 4 +-- .../README.md | 0 .../src/lib.rs | 16 +++++----- 9 files changed, 46 insertions(+), 41 deletions(-) rename primitives/{storage-proof => transaction-storage-proof}/Cargo.toml (92%) rename primitives/{storage-proof => transaction-storage-proof}/README.md (100%) rename primitives/{storage-proof => transaction-storage-proof}/src/lib.rs (94%) diff --git a/Cargo.lock b/Cargo.lock index 310cef0a8a912..88deb673f2856 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5620,7 +5620,7 @@ dependencies = [ "sp-io", "sp-runtime", "sp-std", - "sp-storage-proof", + "sp-transaction-storage-proof", ] [[package]] @@ -9262,21 +9262,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "sp-storage-proof" -version = "3.0.0" -dependencies = [ - "log", - "parity-scale-codec", - "sc-client-api", - "sp-consensus", - "sp-core", - "sp-inherents", - "sp-runtime", - "sp-std", - "sp-trie", -] - [[package]] name = "sp-tasks" version = "3.0.0" @@ -9350,6 +9335,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sp-transaction-storage-proof" +version = "3.0.0" +dependencies = [ + "log", + "parity-scale-codec", + "sc-client-api", + "sp-consensus", + "sp-core", + "sp-inherents", + "sp-runtime", + "sp-std", + "sp-trie", +] + [[package]] name = "sp-trie" version = "3.0.0" diff --git a/Cargo.toml b/Cargo.toml index 02daa6e5ce6a4..f7552f0bbbc48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,12 +176,12 @@ members = [ "primitives/state-machine", "primitives/std", "primitives/storage", - "primitives/storage-proof", "primitives/tasks", "primitives/test-primitives", "primitives/timestamp", "primitives/tracing", "primitives/transaction-pool", + "primitives/transaction-storage-proof", "primitives/trie", "primitives/utils", "primitives/version", diff --git a/frame/transaction-storage/Cargo.toml b/frame/transaction-storage/Cargo.toml index 3e11b60e38706..5f0b1f7b8a5a0 100644 --- a/frame/transaction-storage/Cargo.toml +++ b/frame/transaction-storage/Cargo.toml @@ -23,13 +23,13 @@ sp-runtime = { version = "3.0.0", default-features = false, path = "../../primit sp-std = { version = "3.0.0", default-features = false, path = "../../primitives/std" } sp-io = { version = "3.0.0", default-features = false, path = "../../primitives/io" } sp-inherents = { version = "3.0.0", default-features = false, path = "../../primitives/inherents" } -sp-storage-proof = { version = "3.0.0", default-features = false, path = "../../primitives/storage-proof" } +sp-transaction-storage-proof = { version = "3.0.0", default-features = false, path = "../../primitives/transaction-storage-proof" } sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false, optional = true } frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] frame-support-test = { version = "3.0.0", path = "../support/test" } -sp-storage-proof = { version = "3.0.0", default-features = true, path = "../../primitives/storage-proof" } +sp-transaction-storage-proof = { version = "3.0.0", default-features = true, path = "../../primitives/transaction-storage-proof" } sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false } [features] diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index 490a81632ce9d..128afa1c72fb3 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -22,6 +22,7 @@ use sp_std::*; use super::*; use sp_runtime::traits::{Zero, One, Bounded}; +use sp_transaction_storage_proof::TransactionStorageProof; use frame_system::{RawOrigin, Pallet as System, EventRecord}; use frame_benchmarking::{benchmarks, whitelisted_caller, impl_benchmark_test_suite}; use frame_support::{traits::{Currency}}; @@ -127,7 +128,7 @@ benchmarks! { run_to_block::(11); let random_hash = [0u8]; let mut encoded_proof = PROOF; - let proof = sp_storage_proof::StorageProof::decode(&mut encoded_proof).unwrap(); + let proof = TransactionStorageProof::decode(&mut encoded_proof).unwrap(); }: check_proof(RawOrigin::None, Some(proof)) verify { assert_last_event::(Event::ProofChecked.into()); diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 4490eb9c0a292..e0004d5fa5c66 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -35,7 +35,11 @@ use sp_std::prelude::*; use sp_std::{result}; use codec::{Encode, Decode}; use sp_runtime::traits::{Saturating, BlakeTwo256, Hash, Zero}; -use sp_storage_proof::{StorageProof, DEFAULT_STORAGE_PERIOD}; +use sp_transaction_storage_proof::{ + TransactionStorageProof, InherentError, + random_chunk, encode_index, + CHUNK_SIZE, INHERENT_IDENTIFIER, DEFAULT_STORAGE_PERIOD, +}; /// A type alias for the balance type from this pallet's point of view. type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; @@ -44,7 +48,6 @@ type BalanceOf = <::Currency as Currency<, - proof: Option, + proof: Option, ) -> DispatchResultWithPostInfo { ensure_none(origin)?; let number = >::block_number(); @@ -219,7 +222,7 @@ pub mod pallet { } let proof = proof.ok_or_else(|| Error::::MissingProof)?; let parent_hash = >::parent_hash(); - let selected_chunk_index = sp_storage_proof::random_chunk(parent_hash.as_ref(), total_chunks); + let selected_chunk_index = random_chunk(parent_hash.as_ref(), total_chunks); total_chunks = 0; let mut t = 0; let (info, chunk_index) = loop { @@ -240,7 +243,7 @@ pub mod pallet { sp_io::trie::blake2_256_verify_proof( info.chunk_root, &proof.proof, - &sp_storage_proof::encode_index(chunk_index), + &encode_index(chunk_index), &proof.chunk, ), Error::::InvalidProof @@ -321,11 +324,11 @@ pub mod pallet { #[pallet::inherent] impl ProvideInherent for Pallet { type Call = Call; - type Error = sp_storage_proof::InherentError; - const INHERENT_IDENTIFIER: InherentIdentifier = sp_storage_proof::INHERENT_IDENTIFIER; + type Error = InherentError; + const INHERENT_IDENTIFIER: InherentIdentifier = INHERENT_IDENTIFIER; fn create_inherent(data: &InherentData) -> Option { - let proof = data.get_data::(&Self::INHERENT_IDENTIFIER).unwrap_or(None); + let proof = data.get_data::(&Self::INHERENT_IDENTIFIER).unwrap_or(None); Some(Call::check_proof(proof)) } diff --git a/frame/transaction-storage/src/tests.rs b/frame/transaction-storage/src/tests.rs index 903c78e845fef..34a0069ccabf8 100644 --- a/frame/transaction-storage/src/tests.rs +++ b/frame/transaction-storage/src/tests.rs @@ -23,6 +23,7 @@ use crate::mock::runtime::*; use super::Pallet as TransactionStorage; use frame_support::{assert_ok, assert_noop}; use frame_system::RawOrigin; +use sp_transaction_storage_proof::registration::build_proof; #[test] fn discards_data() { @@ -82,7 +83,7 @@ fn checks_proof() { )); run_to_block::(10); let parent_hash = >::parent_hash(); - let proof = sp_storage_proof::registration::build_proof( + let proof = build_proof( parent_hash.as_ref(), vec![vec![0u8; MAX_DATA_SIZE as usize]] ).unwrap(); @@ -94,12 +95,12 @@ fn checks_proof() { ); run_to_block::(11); let parent_hash = >::parent_hash(); - let proof = sp_storage_proof::registration::build_proof( + let proof = build_proof( parent_hash.as_ref(), vec![vec![0u8; MAX_DATA_SIZE as usize]] ).unwrap(); assert_ok!(TransactionStorage::::check_proof(Origin::none(), Some(proof))); - let invalid_proof = sp_storage_proof::registration::build_proof( + let invalid_proof = build_proof( parent_hash.as_ref(), vec![vec![0u8; 1000]] ).unwrap(); diff --git a/primitives/storage-proof/Cargo.toml b/primitives/transaction-storage-proof/Cargo.toml similarity index 92% rename from primitives/storage-proof/Cargo.toml rename to primitives/transaction-storage-proof/Cargo.toml index e5fc1c3f725a4..fcb003d028329 100644 --- a/primitives/storage-proof/Cargo.toml +++ b/primitives/transaction-storage-proof/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "sp-storage-proof" +name = "sp-transaction-storage-proof" version = "3.0.0" authors = ["Parity Technologies "] -description = "Storage proof primitives" +description = "Transaction storage proof primitives" edition = "2018" license = "Apache-2.0" homepage = "https://substrate.dev" diff --git a/primitives/storage-proof/README.md b/primitives/transaction-storage-proof/README.md similarity index 100% rename from primitives/storage-proof/README.md rename to primitives/transaction-storage-proof/README.md diff --git a/primitives/storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs similarity index 94% rename from primitives/storage-proof/src/lib.rs rename to primitives/transaction-storage-proof/src/lib.rs index afd7a1f826cdf..1585d1ff404e7 100644 --- a/primitives/storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -47,7 +47,7 @@ impl IsFatalError for InherentError { } #[derive(Encode, Decode, Clone, PartialEq, Debug)] -pub struct StorageProof { +pub struct TransactionStorageProof { /// Data chunk that is proved to exist. pub chunk: Vec, /// Trie nodes that compose the proof. @@ -55,13 +55,13 @@ pub struct StorageProof { } /// Auxiliary trait to extract storage proof. -pub trait StorageProofInherentData { +pub trait TransactionStorageProofInherentData { /// Get the proof. - fn storage_proof(&self) -> Result, Error>; + fn storage_proof(&self) -> Result, Error>; } -impl StorageProofInherentData for InherentData { - fn storage_proof(&self) -> Result, Error> { +impl TransactionStorageProofInherentData for InherentData { + fn storage_proof(&self) -> Result, Error> { Ok(self.get_data(&INHERENT_IDENTIFIER)?) } } @@ -81,7 +81,7 @@ impl InherentDataProvider { #[cfg(feature = "std")] impl sp_inherents::ProvideInherentData for InherentDataProvider -where F: Fn() -> Result, Error> +where F: Fn() -> Result, Error> { fn inherent_identifier(&self) -> &'static InherentIdentifier { &INHERENT_IDENTIFIER @@ -178,7 +178,7 @@ pub mod registration { /// Build a proof for a given source of randomness and indexed transactions. pub fn build_proof(random_hash: &[u8], transactions: Vec>) - -> Result + -> Result { let mut db = sp_trie::MemoryDB::::default(); @@ -221,7 +221,7 @@ pub mod registration { } }; - Ok(StorageProof { + Ok(TransactionStorageProof { proof: chunk_proof, chunk: target_chunk.unwrap(), }) From d9203f3d112f3bb00fae1d57d29ed6a645ce1d3a Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Sun, 25 Apr 2021 11:38:19 +0300 Subject: [PATCH 07/20] Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/transaction-storage/src/lib.rs | 6 ++---- frame/transaction-storage/src/mock.rs | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index e0004d5fa5c66..450ca428d3329 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -147,15 +147,13 @@ pub mod pallet { sp_io::transaction_index::index(extrinsic_index, data.len() as u32, content_hash); let block_num = >::block_number(); - let mut counter = >::get().unwrap_or(0); >::insert(block_num, counter, TransactionInfo { chunk_root: root, size: data.len() as u32, content_hash: content_hash.into(), }); + let counter = >::mutate(|c| { *c += 1; c }); Self::deposit_event(Event::Stored(counter)); - counter += 1; - >::put(counter); Ok(().into()) } @@ -293,7 +291,7 @@ pub mod pallet { pub(super) type ByteFee = StorageValue<_, BalanceOf>; #[pallet::storage] - pub(super) type Counter = StorageValue<_, u32>; + pub(super) type Counter = StorageValue<_, u32, ValueQuery>; /// Storage period for data in blocks. Should match `sp_storage_proof::DEFAULT_STORAGE_PERIOD` /// for block authoring. diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index c91e9afcfe619..706c657d4af87 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -36,7 +36,6 @@ pub mod runtime { type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; pub type Block = frame_system::mocking::MockBlock; - // Configure a mock runtime to test the pallet. frame_support::construct_runtime!( pub enum Test where @@ -121,7 +120,7 @@ pub mod runtime { pallet_balances: pallet_balances::GenesisConfig:: { balances: vec![(1, 1000000000), (2, 100), (3, 100), (4, 100)] }, - pallet_transaction_storage: pallet_transaction_storage::GenesisConfig::default(), + pallet_transaction_storage: Default::default(), }.build_storage().unwrap(); t.into() } @@ -145,4 +144,3 @@ pub fn setup() -> Result<(), &'static str> { pallet_transaction_storage::StoragePeriod::::set(Some(storage_period)); Ok(()) } - From 448f024a022f81272cf2f83359b0ecf3040de911 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 25 Apr 2021 11:40:19 +0300 Subject: [PATCH 08/20] Added weight for on_finalize --- frame/transaction-storage/src/benchmarking.rs | 29 +++++++++++++-- frame/transaction-storage/src/lib.rs | 18 +++++++--- frame/transaction-storage/src/weights.rs | 35 ++++++++++++++----- 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index 128afa1c72fb3..6de892300a2a2 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -25,7 +25,7 @@ use sp_runtime::traits::{Zero, One, Bounded}; use sp_transaction_storage_proof::TransactionStorageProof; use frame_system::{RawOrigin, Pallet as System, EventRecord}; use frame_benchmarking::{benchmarks, whitelisted_caller, impl_benchmark_test_suite}; -use frame_support::{traits::{Currency}}; +use frame_support::{traits::{Currency, OnFinalize}}; use crate::Pallet as TransactionStorage; use crate::mock::{run_to_block, setup}; @@ -107,7 +107,7 @@ benchmarks! { T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), - vec![0u8; MAX_DATA_SIZE as usize] + vec![0u8; MAX_DATA_SIZE as usize], )?; }: _(RawOrigin::Signed(caller.clone()), T::BlockNumber::zero(), 0) verify { @@ -122,7 +122,7 @@ benchmarks! { for _ in 0 .. MAX_EXTRINSICS { TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), - vec![0u8; MAX_DATA_SIZE as usize] + vec![0u8; MAX_DATA_SIZE as usize], )?; } run_to_block::(11); @@ -133,6 +133,29 @@ benchmarks! { verify { assert_last_event::(Event::ProofChecked.into()); } + + on_finalize { + let l in 0 .. MAX_EXTRINSICS; + setup::()?; + run_to_block::(1); + let caller: T::AccountId = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + for _ in 0 .. l { + TransactionStorage::::store( + RawOrigin::Signed(caller.clone()).into(), + vec![0u8], + )?; + } + run_to_block::(11); + if l != 0 { + assert!(TransactionRoots::::get(T::BlockNumber::one(), 0).is_some()); + } + }: { + >::on_finalize(frame_system::Pallet::::block_number()) + } + verify { + assert_eq!(TransactionRoots::::get(T::BlockNumber::one(), 0), None); + } } impl_benchmark_test_suite!( diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 450ca428d3329..13375f591b57d 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -52,10 +52,14 @@ pub use weights::WeightInfo; // Increasing it further also requires raising the allocator limit. pub const MAX_DATA_SIZE: u32 = 8 * 1024 * 1024; +/// State data for a stored transaction. #[derive(Encode, Decode, sp_runtime::RuntimeDebug, PartialEq, Eq)] pub struct TransactionInfo { + /// Chunk trie root. chunk_root: ::Output, + /// Plain hash of indexed data. content_hash: ::Output, + /// Size of indexed data in bytes. size: u32, } @@ -103,10 +107,16 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { - fn on_initialize(_n: T::BlockNumber) -> Weight { - // This will be cleared in in `on_finalize` - >::put(0); - 0 + fn on_initialize(n: T::BlockNumber) -> Weight { + let period = >::get().unwrap_or(DEFAULT_STORAGE_PERIOD.into()); + let obsolete = n.saturating_sub(period); + let transaction_count = if obsolete > Zero::zero() { + >::get(obsolete) + } else { + 0 + }; + // return weight for `on_finalize` does + T::WeightInfo::on_finalize(transaction_count) } fn on_finalize(n: T::BlockNumber) { diff --git a/frame/transaction-storage/src/weights.rs b/frame/transaction-storage/src/weights.rs index b20d22c30e5c3..abc17118c9ea5 100644 --- a/frame/transaction-storage/src/weights.rs +++ b/frame/transaction-storage/src/weights.rs @@ -18,16 +18,18 @@ //! Autogenerated weights for pallet_transaction_storage //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-04-15, STEPS: `[10, ]`, REPEAT: 10, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-04-23, STEPS: `[20, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: -// target/release/substrate +// ./target/release/substrate // benchmark -// --chain=dev -// --steps=10 -// --repeat=10 -// --pallet=pallet_transaction-storage +// --chain +// dev +// --steps +// 20 +// --repeat=20 +// --pallet=pallet_transaction_storage // --extrinsic=* // --execution=wasm // --wasm-execution=compiled @@ -47,6 +49,7 @@ pub trait WeightInfo { fn store(l: u32, ) -> Weight; fn renew() -> Weight; fn check_proof_max() -> Weight; + fn on_finalize(l: u32, ) -> Weight; } /// Weights for pallet_transaction_storage using the Substrate node and recommended hardware. @@ -65,9 +68,17 @@ impl WeightInfo for SubstrateWeight { .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn check_proof_max() -> Weight { - (1_482_000_000 as Weight) + (1_495_000_000 as Weight) .saturating_add(T::DbWeight::get().reads(103 as Weight)) } + fn on_finalize(l: u32, ) -> Weight { + (35_169_000 as Weight) + // Standard Error: 7_000 + .saturating_add((1_773_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(l as Weight))) + } } // For backwards compatibility and tests @@ -85,7 +96,15 @@ impl WeightInfo for () { .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn check_proof_max() -> Weight { - (1_482_000_000 as Weight) + (1_495_000_000 as Weight) .saturating_add(RocksDbWeight::get().reads(103 as Weight)) } + fn on_finalize(l: u32, ) -> Weight { + (35_169_000 as Weight) + // Standard Error: 7_000 + .saturating_add((1_773_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(l as Weight))) + } } From ae7ee0fb9272985c0b91ed588c59e7bb6b45c78c Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 25 Apr 2021 11:47:37 +0300 Subject: [PATCH 09/20] Fixed counter mutations --- frame/transaction-storage/src/lib.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 13375f591b57d..1b5b8b7d7977b 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -120,7 +120,7 @@ pub mod pallet { } fn on_finalize(n: T::BlockNumber) { - >::insert(n, >::get().unwrap_or(0)); + >::insert(n, >::get()); >::kill(); // Drop obsolete roots let period = >::get().unwrap_or(DEFAULT_STORAGE_PERIOD.into()); @@ -157,12 +157,12 @@ pub mod pallet { sp_io::transaction_index::index(extrinsic_index, data.len() as u32, content_hash); let block_num = >::block_number(); + let counter = >::mutate(|c| { *c += 1; *c - 1 }); >::insert(block_num, counter, TransactionInfo { chunk_root: root, size: data.len() as u32, content_hash: content_hash.into(), }); - let counter = >::mutate(|c| { *c += 1; c }); Self::deposit_event(Event::Stored(counter)); Ok(().into()) } @@ -186,12 +186,10 @@ pub mod pallet { let extrinsic_index = >::extrinsic_index().unwrap(); sp_io::transaction_index::renew(extrinsic_index, info.content_hash.into()); - let mut counter = >::get().unwrap_or(0); + let counter = >::mutate(|c| { *c += 1; *c - 1 }); let block_num = >::block_number(); >::insert(block_num, counter, info); Self::deposit_event(Event::Renewed(counter)); - counter += 1; - >::put(counter); Ok(().into()) } From 2f616eb2c35d9e7230d62aad20d100abebd40539 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 25 Apr 2021 17:16:01 +0300 Subject: [PATCH 10/20] Reorganized tests --- bin/node/cli/src/chain_spec.rs | 1 + bin/node/runtime/src/lib.rs | 2 +- frame/transaction-storage/Cargo.toml | 2 - frame/transaction-storage/src/benchmarking.rs | 29 +-- frame/transaction-storage/src/lib.rs | 18 +- frame/transaction-storage/src/mock.rs | 198 ++++++++---------- frame/transaction-storage/src/tests.rs | 27 +-- 7 files changed, 127 insertions(+), 150 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index eb3ee5124ac0a..3454aa83c24d4 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -335,6 +335,7 @@ pub fn testnet_genesis( }, pallet_vesting: Default::default(), pallet_gilt: Default::default(), + pallet_transaction_storage: Default::default(), } } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 3094a5bb3ddb0..39e4ab2e03f70 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1166,7 +1166,7 @@ construct_runtime!( Lottery: pallet_lottery::{Pallet, Call, Storage, Event}, Gilt: pallet_gilt::{Pallet, Call, Storage, Event, Config}, Uniques: pallet_uniques::{Pallet, Call, Storage, Event}, - TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Event}, + TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Config, Event}, } ); diff --git a/frame/transaction-storage/Cargo.toml b/frame/transaction-storage/Cargo.toml index 5f0b1f7b8a5a0..8892e234d436f 100644 --- a/frame/transaction-storage/Cargo.toml +++ b/frame/transaction-storage/Cargo.toml @@ -24,7 +24,6 @@ sp-std = { version = "3.0.0", default-features = false, path = "../../primitives sp-io = { version = "3.0.0", default-features = false, path = "../../primitives/io" } sp-inherents = { version = "3.0.0", default-features = false, path = "../../primitives/inherents" } sp-transaction-storage-proof = { version = "3.0.0", default-features = false, path = "../../primitives/transaction-storage-proof" } -sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false, optional = true } frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] @@ -36,7 +35,6 @@ sp-core = { version = "3.0.0", path = "../../primitives/core", default-features default = ["std"] runtime-benchmarks = [ "frame-benchmarking", - "sp-core", "hex-literal", ] std = [ diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index 6de892300a2a2..50b20311985b8 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -25,10 +25,9 @@ use sp_runtime::traits::{Zero, One, Bounded}; use sp_transaction_storage_proof::TransactionStorageProof; use frame_system::{RawOrigin, Pallet as System, EventRecord}; use frame_benchmarking::{benchmarks, whitelisted_caller, impl_benchmark_test_suite}; -use frame_support::{traits::{Currency, OnFinalize}}; +use frame_support::{traits::{Currency, OnFinalize, OnInitialize}}; use crate::Pallet as TransactionStorage; -use crate::mock::{run_to_block, setup}; const MAX_EXTRINSICS: u32 = 100; @@ -89,10 +88,19 @@ fn assert_last_event(generic_event: ::Event) { assert_eq!(event, &system_event); } +pub fn run_to_block(n: T::BlockNumber) { + while frame_system::Pallet::::block_number() < n { + crate::Pallet::::on_finalize(frame_system::Pallet::::block_number()); + frame_system::Pallet::::on_finalize(frame_system::Pallet::::block_number()); + frame_system::Pallet::::set_block_number(frame_system::Pallet::::block_number() + One::one()); + frame_system::Pallet::::on_initialize(frame_system::Pallet::::block_number()); + crate::Pallet::::on_initialize(frame_system::Pallet::::block_number()); + } +} + benchmarks! { store { let l in 1 .. MAX_DATA_SIZE; - setup::()?; let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); }: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize]) @@ -102,7 +110,6 @@ benchmarks! { } renew { - setup::()?; let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); TransactionStorage::::store( @@ -115,8 +122,7 @@ benchmarks! { } check_proof_max { - setup::()?; - run_to_block::(1); + run_to_block::(1u32.into()); let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); for _ in 0 .. MAX_EXTRINSICS { @@ -125,7 +131,7 @@ benchmarks! { vec![0u8; MAX_DATA_SIZE as usize], )?; } - run_to_block::(11); + run_to_block::(StoragePeriod::::get() + T::BlockNumber::one()); let random_hash = [0u8]; let mut encoded_proof = PROOF; let proof = TransactionStorageProof::decode(&mut encoded_proof).unwrap(); @@ -136,8 +142,7 @@ benchmarks! { on_finalize { let l in 0 .. MAX_EXTRINSICS; - setup::()?; - run_to_block::(1); + run_to_block::(1u32.into()); let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); for _ in 0 .. l { @@ -146,7 +151,7 @@ benchmarks! { vec![0u8], )?; } - run_to_block::(11); + run_to_block::(StoragePeriod::::get() + T::BlockNumber::one()); if l != 0 { assert!(TransactionRoots::::get(T::BlockNumber::one(), 0).is_some()); } @@ -160,6 +165,6 @@ benchmarks! { impl_benchmark_test_suite!( TransactionStorage, - crate::mock::runtime::new_test_ext(), - crate::mock::runtime::Test, + crate::mock::new_test_ext(), + crate::mock::Test, ); diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 1b5b8b7d7977b..dea5a640f6e11 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -20,12 +20,13 @@ // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(any(test, feature = "runtime-benchmarks"))] +mod benchmarking; +pub mod weights; + +#[cfg(test)] mod mock; #[cfg(test)] mod tests; -mod benchmarking; -pub mod weights; use frame_support::{ traits::{ReservableCurrency, Currency}, @@ -108,7 +109,7 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(n: T::BlockNumber) -> Weight { - let period = >::get().unwrap_or(DEFAULT_STORAGE_PERIOD.into()); + let period = >::get(); let obsolete = n.saturating_sub(period); let transaction_count = if obsolete > Zero::zero() { >::get(obsolete) @@ -123,7 +124,7 @@ pub mod pallet { >::insert(n, >::get()); >::kill(); // Drop obsolete roots - let period = >::get().unwrap_or(DEFAULT_STORAGE_PERIOD.into()); + let period = >::get(); let obsolete = n.saturating_sub(period); if obsolete > Zero::zero() { >::remove_prefix(obsolete); @@ -207,7 +208,7 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { ensure_none(origin)?; let number = >::block_number(); - let period = >::get().unwrap_or(DEFAULT_STORAGE_PERIOD.into()); + let period = >::get(); let target_number = number.saturating_sub(period); if target_number.is_zero() { ensure!(proof.is_none(), Error::::UnexpectedProof); @@ -304,11 +305,12 @@ pub mod pallet { /// Storage period for data in blocks. Should match `sp_storage_proof::DEFAULT_STORAGE_PERIOD` /// for block authoring. #[pallet::storage] - pub(super) type StoragePeriod = StorageValue<_, T::BlockNumber>; + pub(super) type StoragePeriod = StorageValue<_, T::BlockNumber, ValueQuery>; #[pallet::genesis_config] pub struct GenesisConfig { pub byte_fee: BalanceOf, + pub storage_period: T::BlockNumber, } #[cfg(feature = "std")] @@ -316,6 +318,7 @@ pub mod pallet { fn default() -> Self { Self { byte_fee: 10u32.into(), + storage_period: DEFAULT_STORAGE_PERIOD.into(), } } } @@ -324,6 +327,7 @@ pub mod pallet { impl GenesisBuild for GenesisConfig { fn build(&self) { >::put(&self.byte_fee); + >::put(&self.storage_period); } } diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index 706c657d4af87..5713b6f267c43 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -18,129 +18,103 @@ //! Test environment for transaction-storage pallet. use crate as pallet_transaction_storage; -use crate::Config; - +use sp_core::H256; +use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::Header, BuildStorage}; use frame_support::{ - traits::{OnInitialize, OnFinalize, Currency}, + parameter_types, + traits::{OnInitialize, OnFinalize}, }; -use sp_runtime::traits::One; - -type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; - -#[cfg(test)] -pub mod runtime { - use super::*; - use sp_core::H256; - use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::Header, BuildStorage}; - use frame_support::parameter_types; - type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; - pub type Block = frame_system::mocking::MockBlock; - // Configure a mock runtime to test the pallet. - frame_support::construct_runtime!( - pub enum Test where - Block = Block, - NodeBlock = Block, - UncheckedExtrinsic = UncheckedExtrinsic, - { - System: frame_system::{Pallet, Call, Config, Storage, Event}, - Balances: pallet_balances::{Pallet, Call, Config, Storage, Event}, - TransactionStorage: pallet_transaction_storage::{ - Pallet, Call, Storage, Config, Inherent, Event - }, - } - ); - - parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const SS58Prefix: u8 = 42; +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +pub type Block = frame_system::mocking::MockBlock; + +// Configure a mock runtime to test the pallet. +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Config, Storage, Event}, + TransactionStorage: pallet_transaction_storage::{ + Pallet, Call, Storage, Config, Inherent, Event + }, } +); - impl frame_system::Config for Test { - type BaseCallFilter = (); - type BlockWeights = (); - type BlockLength = (); - type Origin = Origin; - type Call = Call; - type Index = u64; - type BlockNumber = u64; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Header = Header; - type Event = Event; - type BlockHashCount = BlockHashCount; - type DbWeight = (); - type Version = (); - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = SS58Prefix; - type OnSetCode = (); - } - - parameter_types! { - pub const ExistentialDeposit: u64 = 1; - } - - impl pallet_balances::Config for Test { - type Balance = u64; - type DustRemoval = (); - type Event = Event; - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type WeightInfo = (); - type MaxLocks = (); - } +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const SS58Prefix: u8 = 42; +} - parameter_types! { - pub IgnoredIssuance: u64 = Balances::total_balance(&0); // Account zero is ignored. - pub const QueueCount: u32 = 3; - pub const MaxQueueLen: u32 = 3; - pub const FifoQueueLen: u32 = 1; - pub const Period: u64 = 3; - pub const MinFreeze: u64 = 2; - pub const IntakePeriod: u64 = 2; - pub const MaxIntakeBids: u32 = 2; - } +impl frame_system::Config for Test { + type BaseCallFilter = (); + type BlockWeights = (); + type BlockLength = (); + type Origin = Origin; + type Call = Call; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = SS58Prefix; + type OnSetCode = (); +} - impl pallet_transaction_storage::Config for Test { - type Event = Event; - type Call = Call; - type Currency = Balances; - type WeightInfo = (); - } +parameter_types! { + pub const ExistentialDeposit: u64 = 1; +} - pub fn new_test_ext() -> sp_io::TestExternalities { - let t = GenesisConfig { - frame_system: Default::default(), - pallet_balances: pallet_balances::GenesisConfig:: { - balances: vec![(1, 1000000000), (2, 100), (3, 100), (4, 100)] - }, - pallet_transaction_storage: Default::default(), - }.build_storage().unwrap(); - t.into() - } +impl pallet_balances::Config for Test { + type Balance = u64; + type DustRemoval = (); + type Event = Event; + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type WeightInfo = (); + type MaxLocks = (); +} +impl pallet_transaction_storage::Config for Test { + type Event = Event; + type Call = Call; + type Currency = Balances; + type WeightInfo = (); } -pub fn run_to_block(n: u32) { - while frame_system::Pallet::::block_number() < (n as u32).into() { - pallet_transaction_storage::Pallet::::on_finalize(frame_system::Pallet::::block_number()); - frame_system::Pallet::::on_finalize(frame_system::Pallet::::block_number()); - frame_system::Pallet::::set_block_number(frame_system::Pallet::::block_number() + One::one()); - frame_system::Pallet::::on_initialize(frame_system::Pallet::::block_number()); - pallet_transaction_storage::Pallet::::on_initialize(frame_system::Pallet::::block_number()); - } +pub fn new_test_ext() -> sp_io::TestExternalities { + let t = GenesisConfig { + frame_system: Default::default(), + pallet_balances: pallet_balances::GenesisConfig:: { + balances: vec![(1, 1000000000), (2, 100), (3, 100), (4, 100)] + }, + pallet_transaction_storage: pallet_transaction_storage::GenesisConfig:: { + storage_period: 10, + byte_fee: 2, + }, + }.build_storage().unwrap(); + t.into() } -pub fn setup() -> Result<(), &'static str> { - let fee = BalanceOf::::from(2u32); - let storage_period = 10u32.into(); - pallet_transaction_storage::ByteFee::::set(Some(fee)); - pallet_transaction_storage::StoragePeriod::::set(Some(storage_period)); - Ok(()) +pub fn run_to_block(n: u64) { + while System::block_number() < n { + TransactionStorage::on_finalize(System::block_number()); + System::on_finalize(System::block_number()); + System::set_block_number(System::block_number() + 1); + System::on_initialize(System::block_number()); + TransactionStorage::on_initialize(System::block_number()); + } } diff --git a/frame/transaction-storage/src/tests.rs b/frame/transaction-storage/src/tests.rs index 34a0069ccabf8..8d0326e42ff36 100644 --- a/frame/transaction-storage/src/tests.rs +++ b/frame/transaction-storage/src/tests.rs @@ -19,7 +19,6 @@ use super::*; use crate::mock::*; -use crate::mock::runtime::*; use super::Pallet as TransactionStorage; use frame_support::{assert_ok, assert_noop}; use frame_system::RawOrigin; @@ -28,8 +27,7 @@ use sp_transaction_storage_proof::registration::build_proof; #[test] fn discards_data() { new_test_ext().execute_with(|| { - setup::().unwrap(); - run_to_block::(1); + run_to_block(1); let caller = 1; assert_ok!(TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), @@ -39,12 +37,12 @@ fn discards_data() { RawOrigin::Signed(caller.clone()).into(), vec![0u8; 2000 as usize] )); - run_to_block::(11); + run_to_block(11); assert!(TransactionRoots::::get(1, 0).is_some()); assert!(TransactionRoots::::get(1, 1).is_some()); assert!(TransactionRoots::::get(1, 2).is_none()); assert_eq!(TransactionCount::::get(1), 2); - run_to_block::(12); + run_to_block(12); assert!(TransactionRoots::::get(1, 0).is_none()); assert!(TransactionRoots::::get(1, 1).is_none()); assert_eq!(TransactionCount::::get(1), 0); @@ -54,8 +52,7 @@ fn discards_data() { #[test] fn burns_fee() { new_test_ext().execute_with(|| { - setup::().unwrap(); - run_to_block::(1); + run_to_block(1); let caller = 1; assert_noop!(TransactionStorage::::store( RawOrigin::Signed(5).into(), @@ -74,14 +71,13 @@ fn burns_fee() { #[test] fn checks_proof() { new_test_ext().execute_with(|| { - setup::().unwrap(); - run_to_block::(1); + run_to_block(1); let caller = 1; assert_ok!(TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), vec![0u8; MAX_DATA_SIZE as usize] )); - run_to_block::(10); + run_to_block(10); let parent_hash = >::parent_hash(); let proof = build_proof( parent_hash.as_ref(), @@ -93,7 +89,7 @@ fn checks_proof() { ), Error::::UnexpectedProof, ); - run_to_block::(11); + run_to_block(11); let parent_hash = >::parent_hash(); let proof = build_proof( parent_hash.as_ref(), @@ -116,25 +112,24 @@ fn checks_proof() { #[test] fn renews_data() { new_test_ext().execute_with(|| { - setup::().unwrap(); - run_to_block::(1); + run_to_block(1); let caller = 1; assert_ok!(TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), vec![0u8; 2000 as usize] )); let info = TransactionRoots::::get(1, 0).unwrap(); - run_to_block::(6); + run_to_block(6); assert_ok!(TransactionStorage::::renew( RawOrigin::Signed(caller.clone()).into(), 1, // block 0, // transaction )); assert_eq!(Balances::free_balance(1), 1_000_000_000 - 4000 * 2); - run_to_block::(16); + run_to_block(16); assert!(TransactionRoots::::get(1, 0).is_none()); assert_eq!(TransactionRoots::::get(6, 0), Some(info)); - run_to_block::(17); + run_to_block(17); assert!(TransactionRoots::::get(6, 0).is_none()); }); } From 5cdaf89e117eacc90815d3feb1cfbe5198deed69 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 25 Apr 2021 19:08:14 +0300 Subject: [PATCH 11/20] Fixed build --- bin/node/testing/src/genesis.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 905c2f4d70bb2..6f884d1f73b62 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -120,5 +120,6 @@ pub fn config_endowed( }, pallet_vesting: Default::default(), pallet_gilt: Default::default(), + pallet_transaction_storage: Default::default(), } } From b65ca3184666575ea17b12bf54ee9d6b43e65765 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 5 May 2021 17:36:13 +0300 Subject: [PATCH 12/20] Update for the new inherent API --- Cargo.lock | 5 +- bin/node-template/node/src/service.rs | 2 +- bin/node/cli/Cargo.toml | 1 + bin/node/cli/src/service.rs | 12 +- client/api/src/client.rs | 20 +-- client/service/Cargo.toml | 1 + client/service/src/client/client.rs | 15 +++ primitives/inherents/src/client_side.rs | 2 +- .../transaction-storage-proof/Cargo.toml | 9 +- .../transaction-storage-proof/src/lib.rs | 127 +++++++++--------- 10 files changed, 104 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88deb673f2856..d7e89f6573ea7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4203,6 +4203,7 @@ dependencies = [ "sp-runtime", "sp-timestamp", "sp-transaction-pool", + "sp-transaction-storage-proof", "sp-trie", "structopt", "substrate-browser-utils", @@ -7974,6 +7975,7 @@ dependencies = [ "sp-state-machine", "sp-tracing", "sp-transaction-pool", + "sp-transaction-storage-proof", "sp-trie", "sp-utils", "sp-version", @@ -9339,10 +9341,9 @@ dependencies = [ name = "sp-transaction-storage-proof" version = "3.0.0" dependencies = [ + "async-trait", "log", "parity-scale-codec", - "sc-client-api", - "sp-consensus", "sp-core", "sp-inherents", "sp-runtime", diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index f504904100765..74c0dd204633a 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -227,7 +227,7 @@ pub fn new_full(mut config: Configuration) -> Result select_chain, block_import, proposer_factory, - create_inherent_data_providers: move |_, ()| async move { + create_inherent_data_providers: move |_, _| async move { let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 9fcd0875e8dca..376139a29dfff 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -57,6 +57,7 @@ sp-keystore = { version = "0.9.0", path = "../../../primitives/keystore" } sp-io = { version = "3.0.0", path = "../../../primitives/io" } sp-consensus = { version = "0.9.0", path = "../../../primitives/consensus/common" } sp-transaction-pool = { version = "3.0.0", path = "../../../primitives/transaction-pool" } +sp-transaction-storage-proof = { version = "3.0.0", path = "../../../primitives/transaction-storage-proof" } # client dependencies sc-client-api = { version = "3.0.0", path = "../../../client/api" } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 42020e6668e42..f11f8c1334340 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -308,7 +308,7 @@ pub fn new_full_base( env: proposer, block_import, sync_oracle: network.clone(), - create_inherent_data_providers: move |parent, ()| { + create_inherent_data_providers: move |parent, number, ()| { let client_clone = client_clone.clone(); async move { let uncles = sc_consensus_uncles::create_uncles_inherent_data_provider( @@ -324,7 +324,13 @@ pub fn new_full_base( slot_duration, ); - Ok((timestamp, slot, uncles)) + let transaction_proof = sp_transaction_storage_proof::registration::new_data_provider( + &*client_clone, + parent, + number + ); + + Ok((timestamp, slot, uncles, transaction_proof)) } }, force_authoring, @@ -496,7 +502,7 @@ pub fn new_light_base( Some(Box::new(justification_import)), client.clone(), select_chain.clone(), - move |_, ()| async move { + move |_, _| async move { let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = diff --git a/client/api/src/client.rs b/client/api/src/client.rs index d5cb43931dca4..79fb4f8844319 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -84,6 +84,16 @@ pub trait BlockBackend { id: &BlockId ) -> sp_blockchain::Result::Extrinsic>>>; + /// Get all indexed transactions for a block, + /// including renewed transactions. + /// + /// Note that this will only fetch transactions + /// that are indexed by the runtime with `storage_index_transaction`. + fn block_indexed_body( + &self, + id: &BlockId, + ) -> sp_blockchain::Result>>>; + /// Get full block by id. fn block(&self, id: &BlockId) -> sp_blockchain::Result>>; @@ -109,16 +119,6 @@ pub trait BlockBackend { fn has_indexed_transaction(&self, hash: &Block::Hash) -> sp_blockchain::Result { Ok(self.indexed_transaction(hash)?.is_some()) } - - /// Get all indexed transactions for a block, - /// including renewed transactions. - /// - /// Note that this will only fetch transactions - /// that are indexed by the runtime with `storage_index_transaction`. - fn block_indexed_body( - &self, - id: &BlockId, - ) -> sp_blockchain::Result>>>; } /// Provide a list of potential uncle headers for a given block. diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index cff05390d7874..6a98cf82f3e55 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -65,6 +65,7 @@ codec = { package = "parity-scale-codec", version = "2.0.0" } sc-executor = { version = "0.9.0", path = "../executor" } sc-transaction-pool = { version = "3.0.0", path = "../transaction-pool" } sp-transaction-pool = { version = "3.0.0", path = "../../primitives/transaction-pool" } +sp-transaction-storage-proof = { version = "3.0.0", path = "../../primitives/transaction-storage-proof" } sc-rpc-server = { version = "3.0.0", path = "../rpc-servers" } sc-rpc = { version = "3.0.0", path = "../rpc" } sc-block-builder = { version = "0.9.0", path = "../block-builder" } diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index b21d6b31aaf89..166817e013384 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -2057,3 +2057,18 @@ impl sp_consensus::block_validation::Chain for Client) } } + +impl sp_transaction_storage_proof::IndexedBody for Client +where + BE: backend::Backend, + E: CallExecutor, + B: BlockT, +{ + fn block_indexed_body( + &self, + number: NumberFor, + ) ->Result>>, sp_inherents::Error> { + self.backend.blockchain().block_indexed_body(BlockId::number(number)) + .map_err(|e| sp_inherents::Error::Application(Box::new(e))) + } +} diff --git a/primitives/inherents/src/client_side.rs b/primitives/inherents/src/client_side.rs index 38639c5de3227..95b1494b291c0 100644 --- a/primitives/inherents/src/client_side.rs +++ b/primitives/inherents/src/client_side.rs @@ -34,7 +34,7 @@ pub trait CreateInherentDataProviders: Send + Sync { /// Create the inherent data providers at the given `parent` block using the given `extra_args`. async fn create_inherent_data_providers( &self, - parent: Block::Hash, + parent: Block::Header, extra_args: ExtraArgs, ) -> Result>; } diff --git a/primitives/transaction-storage-proof/Cargo.toml b/primitives/transaction-storage-proof/Cargo.toml index fcb003d028329..bbdcb9f989f0b 100644 --- a/primitives/transaction-storage-proof/Cargo.toml +++ b/primitives/transaction-storage-proof/Cargo.toml @@ -16,13 +16,11 @@ targets = ["x86_64-unknown-linux-gnu"] sp-inherents = { version = "3.0.0", default-features = false, path = "../inherents" } sp-runtime = { version = "3.0.0", default-features = false, path = "../runtime" } sp-std = { version = "3.0.0", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } - -sc-client-api = { version = "3.0.0", path = "../../client/api", optional = true } -sp-consensus = { version = "0.9.0", path = "../consensus/common", optional = true } sp-trie = { version = "3.0.0", optional = true, path = "../trie" } sp-core = { version = "3.0.0", path = "../core", optional = true } +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } log = { version = "0.4.8", optional = true } +async-trait = { version = "0.1.48", optional = true } [features] default = [ "std" ] @@ -32,8 +30,7 @@ std = [ "sp-inherents/std", "sp-runtime/std", "sp-trie/std", - "sp-consensus", "sp-core", - "sc-client-api", "log", + "async-trait", ] diff --git a/primitives/transaction-storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs index 1585d1ff404e7..af8615dcbdb63 100644 --- a/primitives/transaction-storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -24,6 +24,7 @@ use sp_std::{result::Result, prelude::*}; use codec::{Encode, Decode}; use sp_inherents::{Error, InherentIdentifier, InherentData, IsFatalError}; +use sp_runtime::{traits::{Block as BlockT, NumberFor}}; /// The identifier for the proof inherent. pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"tx_proof"; @@ -68,36 +69,40 @@ impl TransactionStorageProofInherentData for InherentData { /// Provider for inherent data. #[cfg(feature = "std")] -pub struct InherentDataProvider { - inner: F, +pub struct InherentDataProvider { + proof: Option, } #[cfg(feature = "std")] -impl InherentDataProvider { - pub fn new(proof_oracle: F) -> Self { - InherentDataProvider { inner: proof_oracle } +impl InherentDataProvider { + pub fn new(proof: Option) -> Self { + InherentDataProvider { proof } } } #[cfg(feature = "std")] -impl sp_inherents::ProvideInherentData for InherentDataProvider -where F: Fn() -> Result, Error> -{ - fn inherent_identifier(&self) -> &'static InherentIdentifier { - &INHERENT_IDENTIFIER - } - +#[async_trait::async_trait] +impl sp_inherents::InherentDataProvider for InherentDataProvider { fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), Error> { - let proof = (self.inner)()?; - if let Some(proof) = proof { - inherent_data.put_data(INHERENT_IDENTIFIER, &proof) + if let Some(proof) = &self.proof { + inherent_data.put_data(INHERENT_IDENTIFIER, proof) } else { Ok(()) } } - fn error_to_string(&self, _error: &[u8]) -> Option { - Some(format!("no further information")) + async fn try_handle_error( + &self, + identifier: &InherentIdentifier, + error: &[u8], + ) -> Option> { + if *identifier != INHERENT_IDENTIFIER { + return None + } + + let error = InherentError::decode(&mut &error[..]).ok()?; + + Some(Err(Error::Application(Box::from(format!("{:?}", error))))) } } @@ -114,14 +119,18 @@ pub fn encode_index(input: u32) -> Vec { codec::Encode::encode(&codec::Compact(input)) } +/// An interface to request indexed data from the client. +pub trait IndexedBody { + fn block_indexed_body( + &self, + number: NumberFor, + ) -> Result>>, sp_inherents::Error>; +} + #[cfg(feature = "std")] pub mod registration { - use sp_consensus::SelectChain; - use sp_inherents::{InherentDataProviders}; use log::warn; - use sc_client_api::{HeaderBackend, BlockBackend}; - use sp_runtime::{traits::{Block as BlockT, Header, Saturating, Zero}, generic::BlockId}; - use std::sync::Arc; + use sp_runtime::{traits::{Block as BlockT, Saturating, Zero, NumberFor}}; use sp_trie::TrieMut; use super::*; @@ -129,51 +138,35 @@ pub mod registration { type TrieLayout = sp_trie::Layout::; /// Register uncles inherent data provider, if not registered already. - pub fn register_storage_proof_inherent_data_provider( - client: Arc, - select_chain: SC, - inherent_data_providers: &InherentDataProviders, - ) -> Result<(), sp_consensus::Error> where - B: BlockT, - C: BlockBackend + HeaderBackend + Send + Sync + 'static, - SC: SelectChain + 'static, + pub fn new_data_provider( + client: &C, + head_number: NumberFor, + parent: &B::Hash, + ) -> Result + where + B: BlockT, + C: IndexedBody, { - if !inherent_data_providers.has_provider(&INHERENT_IDENTIFIER) { - inherent_data_providers - .register_provider(InherentDataProvider::new(move || { - { - let chain_head = match select_chain.best_chain() { - Ok(x) => x, - Err(e) => { - warn!(target: "storage-proof", "Unable to get chain head: {:?}", e); - return Ok(None); - } - }; - - let number = chain_head.number().saturating_sub(DEFAULT_STORAGE_PERIOD.into()); - if number.is_zero() { - // Too early to collect proofs. - return Ok(None); - } - - match client.block_indexed_body(&BlockId::number(number)) { - Ok(Some(transactions)) => { - Ok(Some(build_proof(chain_head.parent_hash().as_ref(), transactions)?)) - }, - Ok(None) => { - // Nothing was indexed in that block. - Ok(None) - } - Err(e) => { - warn!(target: "storage-proof", "Unable to get transactions: {:?}", e); - Ok(None) - } - } - } - })) - .map_err(|err| sp_consensus::Error::InherentData(err.into()))?; + let number = head_number.saturating_sub(DEFAULT_STORAGE_PERIOD.into()); + if number.is_zero() { + // Too early to collect proofs. + return Ok(InherentDataProvider::new(None)); } - Ok(()) + + let proof = match client.block_indexed_body(number) { + Ok(Some(transactions)) => { + Some(build_proof(parent.as_ref(), transactions)?) + }, + Ok(None) => { + // Nothing was indexed in that block. + None + } + Err(e) => { + warn!(target: "storage-proof", "Unable to get transactions: {:?}", e); + None + } + }; + Ok(InherentDataProvider::new(proof)) } /// Build a proof for a given source of randomness and indexed transactions. @@ -202,7 +195,7 @@ pub mod registration { for (index, chunk) in chunks.enumerate() { let index = encode_index(index as u32); trie.insert(&index, &chunk) - .map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; + .map_err(|e| sp_inherents::Error::Application(Box::new(e)))?; if chunk_index == target_chunk_index { target_chunk = Some(chunk); target_chunk_key = index; @@ -217,7 +210,7 @@ pub mod registration { &db, transaction_root.clone(), &[target_chunk_key.clone()] - ).map_err(|e| sp_inherents::Error::from(format!("Trie error: {:?}", e)))?; + ).map_err(|e| sp_inherents::Error::Application(Box::new(e)))?; } }; From 6a2aa62f0f5a305e2610ccec3355bbc6d76b544d Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 7 May 2021 11:28:06 +0300 Subject: [PATCH 13/20] Reworked for the new inherents API --- Cargo.lock | 1 - bin/node-template/node/src/service.rs | 2 +- bin/node/cli/Cargo.toml | 1 - bin/node/cli/src/service.rs | 12 ++---- client/service/src/client/client.rs | 12 +++++- frame/transaction-storage/README.md | 3 ++ frame/transaction-storage/src/benchmarking.rs | 2 +- frame/transaction-storage/src/lib.rs | 4 +- frame/transaction-storage/src/tests.rs | 2 +- primitives/inherents/src/client_side.rs | 2 +- .../transaction-storage-proof/src/lib.rs | 38 ++++++++++--------- 11 files changed, 43 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7e89f6573ea7..97b64e07e4133 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4203,7 +4203,6 @@ dependencies = [ "sp-runtime", "sp-timestamp", "sp-transaction-pool", - "sp-transaction-storage-proof", "sp-trie", "structopt", "substrate-browser-utils", diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 74c0dd204633a..f504904100765 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -227,7 +227,7 @@ pub fn new_full(mut config: Configuration) -> Result select_chain, block_import, proposer_factory, - create_inherent_data_providers: move |_, _| async move { + create_inherent_data_providers: move |_, ()| async move { let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 376139a29dfff..9fcd0875e8dca 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -57,7 +57,6 @@ sp-keystore = { version = "0.9.0", path = "../../../primitives/keystore" } sp-io = { version = "3.0.0", path = "../../../primitives/io" } sp-consensus = { version = "0.9.0", path = "../../../primitives/consensus/common" } sp-transaction-pool = { version = "3.0.0", path = "../../../primitives/transaction-pool" } -sp-transaction-storage-proof = { version = "3.0.0", path = "../../../primitives/transaction-storage-proof" } # client dependencies sc-client-api = { version = "3.0.0", path = "../../../client/api" } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index f11f8c1334340..42020e6668e42 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -308,7 +308,7 @@ pub fn new_full_base( env: proposer, block_import, sync_oracle: network.clone(), - create_inherent_data_providers: move |parent, number, ()| { + create_inherent_data_providers: move |parent, ()| { let client_clone = client_clone.clone(); async move { let uncles = sc_consensus_uncles::create_uncles_inherent_data_provider( @@ -324,13 +324,7 @@ pub fn new_full_base( slot_duration, ); - let transaction_proof = sp_transaction_storage_proof::registration::new_data_provider( - &*client_clone, - parent, - number - ); - - Ok((timestamp, slot, uncles, transaction_proof)) + Ok((timestamp, slot, uncles)) } }, force_authoring, @@ -502,7 +496,7 @@ pub fn new_light_base( Some(Box::new(justification_import)), client.clone(), select_chain.clone(), - move |_, _| async move { + move |_, ()| async move { let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 166817e013384..06d9aec4e4fd3 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -2067,8 +2067,16 @@ where fn block_indexed_body( &self, number: NumberFor, - ) ->Result>>, sp_inherents::Error> { + ) ->Result>>, sp_transaction_storage_proof::Error> { self.backend.blockchain().block_indexed_body(BlockId::number(number)) - .map_err(|e| sp_inherents::Error::Application(Box::new(e))) + .map_err(|e| sp_transaction_storage_proof::Error::Application(Box::new(e))) + } + + fn number( + &self, + hash: B::Hash, + ) -> Result>, sp_transaction_storage_proof::Error> { + self.backend.blockchain().number(hash) + .map_err(|e| sp_transaction_storage_proof::Error::Application(Box::new(e))) } } diff --git a/frame/transaction-storage/README.md b/frame/transaction-storage/README.md index 4eaddae1786e7..acdd26a350664 100644 --- a/frame/transaction-storage/README.md +++ b/frame/transaction-storage/README.md @@ -1,2 +1,5 @@ +# Transaction Storage Pallet + +Indexes transactions and manages storage proofs. License: Apache-2.0 diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index 50b20311985b8..0e1c0200fc91a 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Benchmarks for transction-storage Pallet +//! Benchmarks for transaction-storage Pallet #![cfg(feature = "runtime-benchmarks")] diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index dea5a640f6e11..5070d0e5bde04 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -135,7 +135,7 @@ pub mod pallet { #[pallet::call] impl Pallet { - /// Index and store data on chain. Minimum data size is 1 bytes, maxmum is `MAX_DATA_SIZE`. + /// Index and store data on chain. Minimum data size is 1 bytes, maximum is `MAX_DATA_SIZE`. /// Data will be removed after `STORAGE_PERIOD` blocks, unless `renew` is called. /// # /// - n*log(n) of data size, as all data is pushed to an in-memory trie. @@ -173,7 +173,7 @@ pub mod pallet { /// Transaction index is emitted in the `Stored` or `Renewed` event. /// Applies same fees as `store`. /// # - /// - Constant with a single DB read. + /// - Constant. /// # #[pallet::weight(T::WeightInfo::renew())] pub(super) fn renew( diff --git a/frame/transaction-storage/src/tests.rs b/frame/transaction-storage/src/tests.rs index 8d0326e42ff36..2a840a1f5faad 100644 --- a/frame/transaction-storage/src/tests.rs +++ b/frame/transaction-storage/src/tests.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Tests for transction-stroage pallet. +//! Tests for transction-storage pallet. use super::*; use crate::mock::*; diff --git a/primitives/inherents/src/client_side.rs b/primitives/inherents/src/client_side.rs index 95b1494b291c0..38639c5de3227 100644 --- a/primitives/inherents/src/client_side.rs +++ b/primitives/inherents/src/client_side.rs @@ -34,7 +34,7 @@ pub trait CreateInherentDataProviders: Send + Sync { /// Create the inherent data providers at the given `parent` block using the given `extra_args`. async fn create_inherent_data_providers( &self, - parent: Block::Header, + parent: Block::Hash, extra_args: ExtraArgs, ) -> Result>; } diff --git a/primitives/transaction-storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs index af8615dcbdb63..ba9207af9bd5a 100644 --- a/primitives/transaction-storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -23,9 +23,11 @@ use sp_std::{result::Result, prelude::*}; use codec::{Encode, Decode}; -use sp_inherents::{Error, InherentIdentifier, InherentData, IsFatalError}; +use sp_inherents::{InherentIdentifier, InherentData, IsFatalError}; use sp_runtime::{traits::{Block as BlockT, NumberFor}}; +pub use sp_inherents::Error; + /// The identifier for the proof inherent. pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"tx_proof"; /// Storage period for data. @@ -124,13 +126,17 @@ pub trait IndexedBody { fn block_indexed_body( &self, number: NumberFor, - ) -> Result>>, sp_inherents::Error>; + ) -> Result>>, Error>; + + fn number( + &self, + hash: B::Hash, + ) -> Result>, Error>; } #[cfg(feature = "std")] pub mod registration { - use log::warn; - use sp_runtime::{traits::{Block as BlockT, Saturating, Zero, NumberFor}}; + use sp_runtime::{traits::{Block as BlockT, Saturating, Zero, One}}; use sp_trie::TrieMut; use super::*; @@ -140,38 +146,36 @@ pub mod registration { /// Register uncles inherent data provider, if not registered already. pub fn new_data_provider( client: &C, - head_number: NumberFor, parent: &B::Hash, - ) -> Result + ) -> Result where B: BlockT, C: IndexedBody, { - let number = head_number.saturating_sub(DEFAULT_STORAGE_PERIOD.into()); + let parent_number = client.number(parent.clone())?.unwrap_or(Zero::zero()); + let number = parent_number + .saturating_add(One::one()) + .saturating_sub(DEFAULT_STORAGE_PERIOD.into()); if number.is_zero() { // Too early to collect proofs. return Ok(InherentDataProvider::new(None)); } - let proof = match client.block_indexed_body(number) { - Ok(Some(transactions)) => { + let proof = match client.block_indexed_body(number)? { + Some(transactions) => { Some(build_proof(parent.as_ref(), transactions)?) }, - Ok(None) => { + None => { // Nothing was indexed in that block. None } - Err(e) => { - warn!(target: "storage-proof", "Unable to get transactions: {:?}", e); - None - } }; Ok(InherentDataProvider::new(proof)) } /// Build a proof for a given source of randomness and indexed transactions. pub fn build_proof(random_hash: &[u8], transactions: Vec>) - -> Result + -> Result { let mut db = sp_trie::MemoryDB::::default(); @@ -195,7 +199,7 @@ pub mod registration { for (index, chunk) in chunks.enumerate() { let index = encode_index(index as u32); trie.insert(&index, &chunk) - .map_err(|e| sp_inherents::Error::Application(Box::new(e)))?; + .map_err(|e| Error::Application(Box::new(e)))?; if chunk_index == target_chunk_index { target_chunk = Some(chunk); target_chunk_key = index; @@ -210,7 +214,7 @@ pub mod registration { &db, transaction_root.clone(), &[target_chunk_key.clone()] - ).map_err(|e| sp_inherents::Error::Application(Box::new(e)))?; + ).map_err(|e| Error::Application(Box::new(e)))?; } }; From 65723ded5c7fd4d453fa8b54aa17f3550433a244 Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Tue, 1 Jun 2021 14:03:47 +0200 Subject: [PATCH 14/20] Apply suggestions from code review Co-authored-by: cheme Co-authored-by: Alexander Popiak Co-authored-by: Shawn Tabrizi --- frame/transaction-storage/README.md | 3 +++ frame/transaction-storage/src/lib.rs | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frame/transaction-storage/README.md b/frame/transaction-storage/README.md index acdd26a350664..a4f77797f5efd 100644 --- a/frame/transaction-storage/README.md +++ b/frame/transaction-storage/README.md @@ -1,5 +1,8 @@ # Transaction Storage Pallet +Indexes transactions and manages storage proofs. +# Transaction Storage Pallet + Indexes transactions and manages storage proofs. License: Apache-2.0 diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 5070d0e5bde04..327b668e7179e 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -49,7 +49,7 @@ type BalanceOf = <::Currency as Currency<, data: Vec, - ) -> DispatchResultWithPostInfo { + ) -> DispatchResult { ensure!(data.len() > 0, Error::::EmptyTransaction); Self::apply_fee(origin, data.len() as u32)?; @@ -165,7 +165,7 @@ pub mod pallet { content_hash: content_hash.into(), }); Self::deposit_event(Event::Stored(counter)); - Ok(().into()) + Ok(()) } /// Renew previously stored data. Parameters are the block number that contains From 9ac001b6b9a999b4bceaaff415ed0238f83351b5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 3 Jun 2021 14:21:39 +0200 Subject: [PATCH 15/20] Store transactions in a Vec --- frame/transaction-storage/src/benchmarking.rs | 35 +-- frame/transaction-storage/src/lib.rs | 222 ++++++++++++------ frame/transaction-storage/src/mock.rs | 10 +- frame/transaction-storage/src/tests.rs | 89 ++++--- frame/transaction-storage/src/weights.rs | 53 ++--- .../transaction-storage-proof/src/lib.rs | 2 +- 6 files changed, 236 insertions(+), 175 deletions(-) diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index 0e1c0200fc91a..a74a083df5fe1 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -29,7 +29,9 @@ use frame_support::{traits::{Currency, OnFinalize, OnInitialize}}; use crate::Pallet as TransactionStorage; -const MAX_EXTRINSICS: u32 = 100; +/// Assuming 16Mb blocks +const MAX_FULL_EXTRINSICS: u32 = 20; +const MAX_DATA_SIZE: u32 = 8 * 1024 * 1024; const PROOF: &[u8] = &hex_literal::hex!(" 0104000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 @@ -105,7 +107,7 @@ benchmarks! { T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); }: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize]) verify { - assert!(TransactionRoots::::get(T::BlockNumber::one(), 0u32).is_some()); + assert!(!BlockTransactions::::get().is_empty()); assert_last_event::(Event::Stored(0).into()); } @@ -116,16 +118,17 @@ benchmarks! { RawOrigin::Signed(caller.clone()).into(), vec![0u8; MAX_DATA_SIZE as usize], )?; + run_to_block::(1u32.into()); }: _(RawOrigin::Signed(caller.clone()), T::BlockNumber::zero(), 0) verify { - assert_last_event::(Event::Renewed(1).into()); + assert_last_event::(Event::Renewed(0).into()); } check_proof_max { run_to_block::(1u32.into()); let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - for _ in 0 .. MAX_EXTRINSICS { + for _ in 0 .. MAX_FULL_EXTRINSICS { TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), vec![0u8; MAX_DATA_SIZE as usize], @@ -135,32 +138,10 @@ benchmarks! { let random_hash = [0u8]; let mut encoded_proof = PROOF; let proof = TransactionStorageProof::decode(&mut encoded_proof).unwrap(); - }: check_proof(RawOrigin::None, Some(proof)) + }: check_proof(RawOrigin::None, proof) verify { assert_last_event::(Event::ProofChecked.into()); } - - on_finalize { - let l in 0 .. MAX_EXTRINSICS; - run_to_block::(1u32.into()); - let caller: T::AccountId = whitelisted_caller(); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - for _ in 0 .. l { - TransactionStorage::::store( - RawOrigin::Signed(caller.clone()).into(), - vec![0u8], - )?; - } - run_to_block::(StoragePeriod::::get() + T::BlockNumber::one()); - if l != 0 { - assert!(TransactionRoots::::get(T::BlockNumber::one(), 0).is_some()); - } - }: { - >::on_finalize(frame_system::Pallet::::block_number()) - } - verify { - assert_eq!(TransactionRoots::::get(T::BlockNumber::one(), 0), None); - } } impl_benchmark_test_suite!( diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 327b668e7179e..f6af75733bc7e 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -35,7 +35,7 @@ use frame_support::{ use sp_std::prelude::*; use sp_std::{result}; use codec::{Encode, Decode}; -use sp_runtime::traits::{Saturating, BlakeTwo256, Hash, Zero}; +use sp_runtime::traits::{Saturating, BlakeTwo256, Hash, Zero, One}; use sp_transaction_storage_proof::{ TransactionStorageProof, InherentError, random_chunk, encode_index, @@ -50,11 +50,12 @@ pub use pallet::*; pub use weights::WeightInfo; /// Maximum bytes that can be stored in one transaction. -// Increasing it further also requires raising the allocator limit. -pub const MAX_DATA_SIZE: u32 = 8 * 1024 * 1024; +// Setting higher limit also requires raising the allocator limit. +pub const DEFAULT_MAX_TRANSACTION_SIZE: u32 = 8 * 1024 * 1024; +pub const DEFAULT_MAX_BLOCK_TRANSACTIONS: u32 = 512; /// State data for a stored transaction. -#[derive(Encode, Decode, sp_runtime::RuntimeDebug, PartialEq, Eq)] +#[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq, Eq)] pub struct TransactionInfo { /// Chunk trie root. chunk_root: ::Output, @@ -62,6 +63,13 @@ pub struct TransactionInfo { content_hash: ::Output, /// Size of indexed data in bytes. size: u32, + /// Total number of chunks added in the block with this transaction. This + /// is used find transactino info by block chunk index using binary search. + block_chunks: u32, +} + +fn num_chunks(bytes: u32) -> u32 { + ((bytes as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32 } #[frame_support::pallet] @@ -100,6 +108,16 @@ pub mod pallet { MissingProof, /// Unable to verify proof becasue state data is missing. MissingStateData, + /// Double proof check in the block. + DoubleCheck, + /// Storage proof was not checked in the block. + ProofNotChecked, + /// Transaction is too large. + TransactionTooLarge, + /// Too many transactions in the block. + TooManyTransactions, + /// Attempted to call `store` outside of block execution. + BadContext, } #[pallet::pallet] @@ -109,33 +127,43 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(n: T::BlockNumber) -> Weight { + // Drop obsolete roots. The proof for `obsolete` will be checked later + // in this block, so we drop `obsolete` - 1. let period = >::get(); - let obsolete = n.saturating_sub(period); - let transaction_count = if obsolete > Zero::zero() { - >::get(obsolete) - } else { - 0 - }; - // return weight for `on_finalize` does - T::WeightInfo::on_finalize(transaction_count) + let obsolete = n.saturating_sub(period.saturating_add(One::one())); + if obsolete > Zero::zero() { + >::remove(obsolete); + >::remove(obsolete); + } + // 2 writes in `on_initialize` and 2 writes + 2 reads in `on_finalize` + T::DbWeight::get().reads_writes(2, 4) } fn on_finalize(n: T::BlockNumber) { - >::insert(n, >::get()); - >::kill(); - // Drop obsolete roots - let period = >::get(); - let obsolete = n.saturating_sub(period); - if obsolete > Zero::zero() { - >::remove_prefix(obsolete); - >::remove(obsolete); + assert!( + >::take() + || { + // Proof is not required for early or empty blocks. + let number = >::block_number(); + let period = >::get(); + let target_number = number.saturating_sub(period); + target_number.is_zero() || >::get(target_number) == 0 + }, + "Storage proof must be checked once in the block" + ); + // Insert new transactions + let transactions = >::take(); + let total_chunks = transactions.last().map_or(0, |t| t.block_chunks); + if total_chunks != 0 { + >::insert(n, total_chunks); + >::insert(n, transactions); } } } #[pallet::call] impl Pallet { - /// Index and store data on chain. Minimum data size is 1 bytes, maximum is `MAX_DATA_SIZE`. + /// Index and store data on chain. Minimum data size is 1 bytes, maximum is `MaxTransactionSize`. /// Data will be removed after `STORAGE_PERIOD` blocks, unless `renew` is called. /// # /// - n*log(n) of data size, as all data is pushed to an in-memory trie. @@ -147,24 +175,35 @@ pub mod pallet { data: Vec, ) -> DispatchResult { ensure!(data.len() > 0, Error::::EmptyTransaction); + ensure!(data.len() <= MaxTransactionSize::::get() as usize, Error::::TransactionTooLarge); Self::apply_fee(origin, data.len() as u32)?; // Chunk data and compute storage root + let chunk_count = num_chunks(data.len() as u32); let chunks = data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); let root = sp_io::trie::blake2_256_ordered_root(chunks); let content_hash = sp_io::hashing::blake2_256(&data); - let extrinsic_index = >::extrinsic_index().unwrap(); + let extrinsic_index = >::extrinsic_index().ok_or_else( + || Error::::BadContext)?; sp_io::transaction_index::index(extrinsic_index, data.len() as u32, content_hash); - let block_num = >::block_number(); - let counter = >::mutate(|c| { *c += 1; *c - 1 }); - >::insert(block_num, counter, TransactionInfo { - chunk_root: root, - size: data.len() as u32, - content_hash: content_hash.into(), - }); - Self::deposit_event(Event::Stored(counter)); + let mut index = 0; + >::mutate(|transactions| { + if transactions.len() + 1 > MaxBlockTransactions::::get() as usize { + return Err(Error::::TooManyTransactions) + } + let total_chunks = transactions.last().map_or(0, |t| t.block_chunks) + chunk_count; + index = transactions.len() as u32; + transactions.push(TransactionInfo { + chunk_root: root, + size: data.len() as u32, + content_hash: content_hash.into(), + block_chunks: total_chunks, + }); + Ok(()) + })?; + Self::deposit_event(Event::Stored(index)); Ok(()) } @@ -181,16 +220,30 @@ pub mod pallet { block: T::BlockNumber, index: u32, ) -> DispatchResultWithPostInfo { - let info = >::get(block, index).ok_or(Error::::RenewedNotFound)?; + let transactions = >::get(block).ok_or(Error::::RenewedNotFound)?; + let info = transactions.get(index as usize).ok_or(Error::::RenewedNotFound)?; Self::apply_fee(origin, info.size)?; let extrinsic_index = >::extrinsic_index().unwrap(); sp_io::transaction_index::renew(extrinsic_index, info.content_hash.into()); - let counter = >::mutate(|c| { *c += 1; *c - 1 }); - let block_num = >::block_number(); - >::insert(block_num, counter, info); - Self::deposit_event(Event::Renewed(counter)); + let mut index = 0; + >::mutate(|transactions| { + if transactions.len() + 1 > MaxBlockTransactions::::get() as usize { + return Err(Error::::TooManyTransactions) + } + let chunks = num_chunks(info.size); + let total_chunks = transactions.last().map_or(0, |t| t.block_chunks) + chunks; + index = transactions.len() as u32; + transactions.push(TransactionInfo { + chunk_root: info.chunk_root, + size: info.size, + content_hash: info.content_hash, + block_chunks: total_chunks, + }); + Ok(()) + })?; + Self::deposit_event(Event::Renewed(index)); Ok(().into()) } @@ -204,48 +257,31 @@ pub mod pallet { #[pallet::weight((T::WeightInfo::check_proof_max(), DispatchClass::Mandatory))] pub(super) fn check_proof( origin: OriginFor, - proof: Option, + proof: TransactionStorageProof, ) -> DispatchResultWithPostInfo { ensure_none(origin)?; + ensure!(!ProofChecked::::get(), Error::::DoubleCheck); let number = >::block_number(); let period = >::get(); let target_number = number.saturating_sub(period); - if target_number.is_zero() { - ensure!(proof.is_none(), Error::::UnexpectedProof); - return Ok(().into()) - } - let transaction_count = >::get(target_number); - let mut total_chunks = 0; - for t in 0 .. transaction_count { - match >::get(target_number, t) { - Some(info) => total_chunks += - ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32, - None => break, - } - } - if total_chunks == 0 { - ensure!(proof.is_none(), Error::::UnexpectedProof); - return Ok(().into()) - } - let proof = proof.ok_or_else(|| Error::::MissingProof)?; + ensure!(!target_number.is_zero(), Error::::UnexpectedProof); + let total_chunks = >::get(target_number); + ensure!(total_chunks != 0, Error::::UnexpectedProof); let parent_hash = >::parent_hash(); let selected_chunk_index = random_chunk(parent_hash.as_ref(), total_chunks); - total_chunks = 0; - let mut t = 0; - let (info, chunk_index) = loop { - match >::get(target_number, t) { - Some(info) => { - let chunks = ((info.size as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32; - if total_chunks + chunks >= selected_chunk_index { - break (info, selected_chunk_index - total_chunks) - } - total_chunks += chunks; - }, - None => Err(Error::::MissingStateData)?, - } - t += 1; + let (info, chunk_index) = match >::get(target_number) { + Some(infos) => { + let index = match infos.binary_search_by_key(&selected_chunk_index, |info| info.block_chunks) { + Ok(index) => index, + Err(index) => index, + }; + let info = infos.get(index).ok_or_else(|| Error::::MissingStateData)?.clone(); + let chunks = num_chunks(info.size); + let prev_chunks = info.block_chunks - chunks; + (info, selected_chunk_index - prev_chunks) + }, + None => Err(Error::::MissingStateData)?, }; - ensure!( sp_io::trie::blake2_256_verify_proof( info.chunk_root, @@ -255,6 +291,7 @@ pub mod pallet { ), Error::::InvalidProof ); + ProofChecked::::put(true); Self::deposit_event(Event::ProofChecked); Ok(().into()) } @@ -271,22 +308,20 @@ pub mod pallet { ProofChecked, } - /// Collection of extrinsic storage roots for the current block. + /// Collection of transaction metadata by block number. #[pallet::storage] #[pallet::getter(fn transaction_roots)] - pub(super) type TransactionRoots = StorageDoubleMap< + pub(super) type Transactions = StorageMap< _, Blake2_128Concat, T::BlockNumber, - Blake2_128Concat, - u32, - TransactionInfo, + Vec, OptionQuery, >; /// Count indexed chunks for each block. #[pallet::storage] - pub(super) type TransactionCount = StorageMap< + pub(super) type ChunkCount = StorageMap< _, Blake2_128Concat, T::BlockNumber, @@ -300,17 +335,41 @@ pub mod pallet { pub(super) type ByteFee = StorageValue<_, BalanceOf>; #[pallet::storage] - pub(super) type Counter = StorageValue<_, u32, ValueQuery>; + #[pallet::getter(fn entry_fee)] + /// Storage fee per transaction. + pub(super) type EntryFee = StorageValue<_, BalanceOf>; + + #[pallet::storage] + #[pallet::getter(fn max_transaction_size)] + /// Maximum data set in a single transaction in bytes. + pub(super) type MaxTransactionSize = StorageValue<_, u32, ValueQuery>; + + #[pallet::storage] + #[pallet::getter(fn max_block_transactions)] + /// Maximum number of indexed transactions in the block. + pub(super) type MaxBlockTransactions = StorageValue<_, u32, ValueQuery>; /// Storage period for data in blocks. Should match `sp_storage_proof::DEFAULT_STORAGE_PERIOD` /// for block authoring. #[pallet::storage] pub(super) type StoragePeriod = StorageValue<_, T::BlockNumber, ValueQuery>; + // Intermediates + #[pallet::storage] + pub(super) type BlockTransactions = StorageValue<_, Vec, ValueQuery>; + + /// Was the proof checked in this block? + #[pallet::storage] + pub(super) type ProofChecked = StorageValue<_, bool, ValueQuery>; + + #[pallet::genesis_config] pub struct GenesisConfig { pub byte_fee: BalanceOf, + pub entry_fee: BalanceOf, pub storage_period: T::BlockNumber, + pub max_block_transactions: u32, + pub max_transaction_size: u32, } #[cfg(feature = "std")] @@ -318,7 +377,10 @@ pub mod pallet { fn default() -> Self { Self { byte_fee: 10u32.into(), + entry_fee: 1000u32.into(), storage_period: DEFAULT_STORAGE_PERIOD.into(), + max_block_transactions: DEFAULT_MAX_BLOCK_TRANSACTIONS, + max_transaction_size: DEFAULT_MAX_TRANSACTION_SIZE, } } } @@ -327,6 +389,9 @@ pub mod pallet { impl GenesisBuild for GenesisConfig { fn build(&self) { >::put(&self.byte_fee); + >::put(&self.entry_fee); + >::put(&self.max_transaction_size); + >::put(&self.max_block_transactions); >::put(&self.storage_period); } } @@ -339,7 +404,7 @@ pub mod pallet { fn create_inherent(data: &InherentData) -> Option { let proof = data.get_data::(&Self::INHERENT_IDENTIFIER).unwrap_or(None); - Some(Call::check_proof(proof)) + proof.map(Call::check_proof) } fn check_inherent(_call: &Self::Call, _data: &InherentData) -> result::Result<(), Self::Error> { @@ -355,7 +420,8 @@ pub mod pallet { fn apply_fee(origin: OriginFor, size: u32) -> DispatchResult { let sender = ensure_signed(origin)?; let byte_fee = ByteFee::::get().ok_or(Error::::NotConfigured)?; - let fee = byte_fee.saturating_mul(size.into()); + let entry_fee = EntryFee::::get().ok_or(Error::::NotConfigured)?; + let fee = byte_fee.saturating_mul(size.into()).saturating_add(entry_fee); ensure!(T::Currency::can_slash(&sender, fee), Error::::InsufficientFunds); T::Currency::slash(&sender, fee); Ok(()) diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index 5713b6f267c43..a5863c2585cee 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -18,6 +18,7 @@ //! Test environment for transaction-storage pallet. use crate as pallet_transaction_storage; +use crate::TransactionStorageProof; use sp_core::H256; use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::Header, BuildStorage}; use frame_support::{ @@ -25,6 +26,7 @@ use frame_support::{ traits::{OnInitialize, OnFinalize}, }; + type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; pub type Block = frame_system::mocking::MockBlock; @@ -104,13 +106,19 @@ pub fn new_test_ext() -> sp_io::TestExternalities { pallet_transaction_storage: pallet_transaction_storage::GenesisConfig:: { storage_period: 10, byte_fee: 2, + entry_fee: 200, + max_block_transactions: crate::DEFAULT_MAX_BLOCK_TRANSACTIONS, + max_transaction_size: crate::DEFAULT_MAX_TRANSACTION_SIZE, }, }.build_storage().unwrap(); t.into() } -pub fn run_to_block(n: u64) { +pub fn run_to_block(n: u64, f: impl Fn() -> Option) { while System::block_number() < n { + if let Some(proof) = f() { + TransactionStorage::check_proof(Origin::none(), proof).unwrap(); + } TransactionStorage::on_finalize(System::block_number()); System::on_finalize(System::block_number()); System::set_block_number(System::block_number() + 1); diff --git a/frame/transaction-storage/src/tests.rs b/frame/transaction-storage/src/tests.rs index 2a840a1f5faad..50594f1bce9dc 100644 --- a/frame/transaction-storage/src/tests.rs +++ b/frame/transaction-storage/src/tests.rs @@ -24,10 +24,12 @@ use frame_support::{assert_ok, assert_noop}; use frame_system::RawOrigin; use sp_transaction_storage_proof::registration::build_proof; +const MAX_DATA_SIZE: u32 = DEFAULT_MAX_TRANSACTION_SIZE; + #[test] fn discards_data() { new_test_ext().execute_with(|| { - run_to_block(1); + run_to_block(1, || None); let caller = 1; assert_ok!(TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), @@ -37,22 +39,30 @@ fn discards_data() { RawOrigin::Signed(caller.clone()).into(), vec![0u8; 2000 as usize] )); - run_to_block(11); - assert!(TransactionRoots::::get(1, 0).is_some()); - assert!(TransactionRoots::::get(1, 1).is_some()); - assert!(TransactionRoots::::get(1, 2).is_none()); - assert_eq!(TransactionCount::::get(1), 2); - run_to_block(12); - assert!(TransactionRoots::::get(1, 0).is_none()); - assert!(TransactionRoots::::get(1, 1).is_none()); - assert_eq!(TransactionCount::::get(1), 0); + let proof_provider = || { + let block_num = >::block_number(); + if block_num == 11 { + let parent_hash = >::parent_hash(); + Some(build_proof(parent_hash.as_ref(), vec![vec![0u8; 2000], vec![0u8; 2000]]).unwrap()) + } else { + None + } + }; + run_to_block(11, proof_provider); + assert!(Transactions::::get(1).is_some()); + let transctions = Transactions::::get(1).unwrap(); + assert_eq!(transctions.len(), 2); + assert_eq!(ChunkCount::::get(1), 16); + run_to_block(12, proof_provider); + assert!(Transactions::::get(1).is_none()); + assert_eq!(ChunkCount::::get(1), 0); }); } #[test] fn burns_fee() { new_test_ext().execute_with(|| { - run_to_block(1); + run_to_block(1, || None); let caller = 1; assert_noop!(TransactionStorage::::store( RawOrigin::Signed(5).into(), @@ -64,20 +74,20 @@ fn burns_fee() { RawOrigin::Signed(caller.clone()).into(), vec![0u8; 2000 as usize] )); - assert_eq!(Balances::free_balance(1), 1_000_000_000 - 2000 * 2); + assert_eq!(Balances::free_balance(1), 1_000_000_000 - 2000 * 2 - 200); }); } #[test] fn checks_proof() { new_test_ext().execute_with(|| { - run_to_block(1); + run_to_block(1, || None); let caller = 1; assert_ok!(TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), vec![0u8; MAX_DATA_SIZE as usize] )); - run_to_block(10); + run_to_block(10, || None); let parent_hash = >::parent_hash(); let proof = build_proof( parent_hash.as_ref(), @@ -85,52 +95,63 @@ fn checks_proof() { ).unwrap(); assert_noop!(TransactionStorage::::check_proof( Origin::none(), - Some(proof), + proof, ), Error::::UnexpectedProof, ); - run_to_block(11); + run_to_block(11, || None); let parent_hash = >::parent_hash(); - let proof = build_proof( - parent_hash.as_ref(), - vec![vec![0u8; MAX_DATA_SIZE as usize]] - ).unwrap(); - assert_ok!(TransactionStorage::::check_proof(Origin::none(), Some(proof))); + let invalid_proof = build_proof( parent_hash.as_ref(), vec![vec![0u8; 1000]] ).unwrap(); assert_noop!(TransactionStorage::::check_proof( Origin::none(), - Some(invalid_proof), - ), - Error::::InvalidProof, + invalid_proof, + ), + Error::::InvalidProof, ); + + let proof = build_proof( + parent_hash.as_ref(), + vec![vec![0u8; MAX_DATA_SIZE as usize]] + ).unwrap(); + assert_ok!(TransactionStorage::::check_proof(Origin::none(), proof)); }); } #[test] fn renews_data() { new_test_ext().execute_with(|| { - run_to_block(1); + run_to_block(1, || None); let caller = 1; assert_ok!(TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), - vec![0u8; 2000 as usize] + vec![0u8; 2000] )); - let info = TransactionRoots::::get(1, 0).unwrap(); - run_to_block(6); + let info = BlockTransactions::::get().last().unwrap().clone(); + run_to_block(6, || None); assert_ok!(TransactionStorage::::renew( RawOrigin::Signed(caller.clone()).into(), 1, // block 0, // transaction )); - assert_eq!(Balances::free_balance(1), 1_000_000_000 - 4000 * 2); - run_to_block(16); - assert!(TransactionRoots::::get(1, 0).is_none()); - assert_eq!(TransactionRoots::::get(6, 0), Some(info)); - run_to_block(17); - assert!(TransactionRoots::::get(6, 0).is_none()); + assert_eq!(Balances::free_balance(1), 1_000_000_000 - 4000 * 2 - 200 * 2); + let proof_provider = || { + let block_num = >::block_number(); + if block_num == 11 || block_num == 16 { + let parent_hash = >::parent_hash(); + Some(build_proof(parent_hash.as_ref(), vec![vec![0u8; 2000]]).unwrap()) + } else { + None + } + }; + run_to_block(16, proof_provider); + assert!(Transactions::::get(1).is_none()); + assert_eq!(Transactions::::get(6).unwrap().get(0), Some(info).as_ref()); + run_to_block(17, proof_provider); + assert!(Transactions::::get(6).is_none()); }); } diff --git a/frame/transaction-storage/src/weights.rs b/frame/transaction-storage/src/weights.rs index abc17118c9ea5..7951db8828d07 100644 --- a/frame/transaction-storage/src/weights.rs +++ b/frame/transaction-storage/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_transaction_storage //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-04-23, STEPS: `[20, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-06-03, STEPS: `[20, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -49,7 +49,6 @@ pub trait WeightInfo { fn store(l: u32, ) -> Weight; fn renew() -> Weight; fn check_proof_max() -> Weight; - fn on_finalize(l: u32, ) -> Weight; } /// Weights for pallet_transaction_storage using the Substrate node and recommended hardware. @@ -58,26 +57,19 @@ impl WeightInfo for SubstrateWeight { fn store(l: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((12_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add((10_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn renew() -> Weight { - (120_000_000 as Weight) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (97_000_000 as Weight) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn check_proof_max() -> Weight { - (1_495_000_000 as Weight) - .saturating_add(T::DbWeight::get().reads(103 as Weight)) - } - fn on_finalize(l: u32, ) -> Weight { - (35_169_000 as Weight) - // Standard Error: 7_000 - .saturating_add((1_773_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) - .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(l as Weight))) + (99_000_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) } } @@ -86,25 +78,18 @@ impl WeightInfo for () { fn store(l: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((12_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add((10_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn renew() -> Weight { - (120_000_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (97_000_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn check_proof_max() -> Weight { - (1_495_000_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(103 as Weight)) - } - fn on_finalize(l: u32, ) -> Weight { - (35_169_000 as Weight) - // Standard Error: 7_000 - .saturating_add((1_773_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(l as Weight))) + (99_000_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } } diff --git a/primitives/transaction-storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs index ba9207af9bd5a..825de27b2a5a9 100644 --- a/primitives/transaction-storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -143,7 +143,7 @@ pub mod registration { type Hasher = sp_core::Blake2Hasher; type TrieLayout = sp_trie::Layout::; - /// Register uncles inherent data provider, if not registered already. + /// Create a new inherent data provider instance for a given parent block hash. pub fn new_data_provider( client: &C, parent: &B::Hash, From 4a9445099a02cf453ede8e5c15e3b87e1a9d113b Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 3 Jun 2021 16:52:46 +0200 Subject: [PATCH 16/20] Added FeeDestination --- frame/transaction-storage/src/lib.rs | 18 ++++++++++++------ frame/transaction-storage/src/mock.rs | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index f6af75733bc7e..0b77ebebdc9fc 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -29,7 +29,7 @@ mod mock; mod tests; use frame_support::{ - traits::{ReservableCurrency, Currency}, + traits::{ReservableCurrency, Currency, OnUnbalanced}, dispatch::{Dispatchable, GetDispatchInfo}, }; use sp_std::prelude::*; @@ -44,6 +44,8 @@ use sp_transaction_storage_proof::{ /// A type alias for the balance type from this pallet's point of view. type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; +type NegativeImbalanceOf = <::Currency as Currency<::AccountId>> + ::NegativeImbalance; // Re-export pallet items so that they can be accessed from the crate namespace. pub use pallet::*; @@ -86,6 +88,8 @@ pub mod pallet { type Call: Parameter + Dispatchable + GetDispatchInfo + From>; /// The currency trait. type Currency: ReservableCurrency; + /// Handler for the unbalanced decrease when fees are burned. + type FeeDestination: OnUnbalanced>; /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } @@ -176,7 +180,8 @@ pub mod pallet { ) -> DispatchResult { ensure!(data.len() > 0, Error::::EmptyTransaction); ensure!(data.len() <= MaxTransactionSize::::get() as usize, Error::::TransactionTooLarge); - Self::apply_fee(origin, data.len() as u32)?; + let sender = ensure_signed(origin)?; + Self::apply_fee(sender, data.len() as u32)?; // Chunk data and compute storage root let chunk_count = num_chunks(data.len() as u32); @@ -220,9 +225,10 @@ pub mod pallet { block: T::BlockNumber, index: u32, ) -> DispatchResultWithPostInfo { + let sender = ensure_signed(origin)?; let transactions = >::get(block).ok_or(Error::::RenewedNotFound)?; let info = transactions.get(index as usize).ok_or(Error::::RenewedNotFound)?; - Self::apply_fee(origin, info.size)?; + Self::apply_fee(sender, info.size)?; let extrinsic_index = >::extrinsic_index().unwrap(); sp_io::transaction_index::renew(extrinsic_index, info.content_hash.into()); @@ -417,13 +423,13 @@ pub mod pallet { } impl Pallet { - fn apply_fee(origin: OriginFor, size: u32) -> DispatchResult { - let sender = ensure_signed(origin)?; + fn apply_fee(sender: T::AccountId, size: u32) -> DispatchResult { let byte_fee = ByteFee::::get().ok_or(Error::::NotConfigured)?; let entry_fee = EntryFee::::get().ok_or(Error::::NotConfigured)?; let fee = byte_fee.saturating_mul(size.into()).saturating_add(entry_fee); ensure!(T::Currency::can_slash(&sender, fee), Error::::InsufficientFunds); - T::Currency::slash(&sender, fee); + let (credit, _) = T::Currency::slash(&sender, fee); + T::FeeDestination::on_unbalanced(credit); Ok(()) } } diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index a5863c2585cee..51eb61dd26b78 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -94,6 +94,7 @@ impl pallet_transaction_storage::Config for Test { type Event = Event; type Call = Call; type Currency = Balances; + type FeeDestination = (); type WeightInfo = (); } From c8c0c84c3fdf37649a6ae3572a2dbbb23c1b147a Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 3 Jun 2021 17:00:08 +0200 Subject: [PATCH 17/20] Get rid of constants --- frame/transaction-storage/src/benchmarking.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index a74a083df5fe1..ffb4d23de119f 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -29,10 +29,6 @@ use frame_support::{traits::{Currency, OnFinalize, OnInitialize}}; use crate::Pallet as TransactionStorage; -/// Assuming 16Mb blocks -const MAX_FULL_EXTRINSICS: u32 = 20; -const MAX_DATA_SIZE: u32 = 8 * 1024 * 1024; - const PROOF: &[u8] = &hex_literal::hex!(" 0104000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 @@ -102,7 +98,7 @@ pub fn run_to_block(n: T::BlockNumber) { benchmarks! { store { - let l in 1 .. MAX_DATA_SIZE; + let l in 1 .. MaxTransactionSize::::get(); let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); }: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize]) @@ -116,7 +112,7 @@ benchmarks! { T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), - vec![0u8; MAX_DATA_SIZE as usize], + vec![0u8; MaxTransactionSize::::get() as usize], )?; run_to_block::(1u32.into()); }: _(RawOrigin::Signed(caller.clone()), T::BlockNumber::zero(), 0) @@ -128,10 +124,10 @@ benchmarks! { run_to_block::(1u32.into()); let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - for _ in 0 .. MAX_FULL_EXTRINSICS { + for _ in 0 .. MaxBlockTransactions::::get() { TransactionStorage::::store( RawOrigin::Signed(caller.clone()).into(), - vec![0u8; MAX_DATA_SIZE as usize], + vec![0u8; MaxTransactionSize::::get() as usize], )?; } run_to_block::(StoragePeriod::::get() + T::BlockNumber::one()); From 0c059ee1d8bfc36a68c449b230aed490e7cb7e7b Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 3 Jun 2021 17:13:45 +0200 Subject: [PATCH 18/20] Fixed node runtime build --- bin/node/runtime/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 39e4ab2e03f70..97975c55e9601 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1118,6 +1118,7 @@ impl pallet_transaction_storage::Config for Runtime { type Event = Event; type Currency = Balances; type Call = Call; + type FeeDestination = (); type WeightInfo = pallet_transaction_storage::weights::SubstrateWeight; } From b18d8551080fb2f04ef5afbb39913f3d3ae0e6d3 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 3 Jun 2021 17:45:17 +0200 Subject: [PATCH 19/20] Fixed benches --- bin/node/bench/src/import.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index 1da284217884a..b4fee58dac025 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -140,7 +140,7 @@ impl core::Benchmark for ImportBenchmark { match self.block_type { BlockType::RandomTransfersKeepAlive => { // should be 5 per signed extrinsic + 1 per unsigned - // we have 2 unsigned and the rest are signed in the block + // we have 1 unsigned and the rest are signed in the block // those 5 events per signed are: // - new account (RawEvent::NewAccount) as we always transfer fund to non-existant account // - endowed (RawEvent::Endowed) for this new account @@ -149,7 +149,7 @@ impl core::Benchmark for ImportBenchmark { // - extrinsic success assert_eq!( node_runtime::System::events().len(), - (self.block.extrinsics.len() - 2) * 5 + 2, + (self.block.extrinsics.len() - 1) * 5 + 1, ); }, BlockType::Noop => { @@ -157,11 +157,11 @@ impl core::Benchmark for ImportBenchmark { node_runtime::System::events().len(), // should be 2 per signed extrinsic + 1 per unsigned - // we have 2 unsigned and the rest are signed in the block + // we have 1 unsigned and the rest are signed in the block // those 2 events per signed are: // - deposit event for charging transaction fee // - extrinsic success - (self.block.extrinsics.len() - 2) * 2 + 2, + (self.block.extrinsics.len() - 1) * 2 + 1, ); }, _ => {}, From bca7a8177ceb5f41ac9f9ee1bda5fafbaa0eb673 Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Thu, 3 Jun 2021 20:12:12 +0200 Subject: [PATCH 20/20] Update frame/transaction-storage/src/lib.rs Co-authored-by: cheme --- frame/transaction-storage/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 0b77ebebdc9fc..ef824a8399f57 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -66,7 +66,7 @@ pub struct TransactionInfo { /// Size of indexed data in bytes. size: u32, /// Total number of chunks added in the block with this transaction. This - /// is used find transactino info by block chunk index using binary search. + /// is used find transaction info by block chunk index using binary search. block_chunks: u32, }