Skip to content
This repository has been archived by the owner on Sep 13, 2022. It is now read-only.

Commit

Permalink
refactor(storage): optimize performance (#296)
Browse files Browse the repository at this point in the history 
* refactor(storage): insert_transactions() now takes block height

* chore(storage): disable apm for better compilation errors

* refactor(stroage): force index in insert_transactions fn signature

* refactor(storage): storage api

Add iter to adapter trait
Remove index from key prefix

* chore(storage): bump rocksdb to 0.14

* change(storage): use sync codec instead

* feat(storage): add iter suppoert to rocksdb api

* feat(storage): implement iter api for both memory and rocksdb adapter

* fix(storage): test code

* feat(storage): bench example

* change(storage): lower NUMBER_OF_TXS_PER_ROUND to 1.5W

* feat(storage): query transaction and receipt by hash

* change(storage): speed up get_transactions

* refactor: use new storage api

* chore: restore apm

* fix: test code

* fix: clippy warnings

* fix: clippy::suspicious-else-formatting on get_receipts

* fix(ci): use rocksdb from crate

* change(consensus): remove unused pull_txs from traits

* perf(storage): speed up get_receipts

* fix(storage): clippy warning on suspicious_else_formatting
  • Loading branch information
@zeroqn
zeroqn authored May 29, 2020
1 parent 53f4b00 commit 49cb7b6
Show file tree
Hide file tree
Showing 35 changed files with 1,375 additions and 2,309 deletions.
2 changes: 0 additions & 2 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ addons:
apt:
packages:
- gcc
- librocksdb5.8
- librocksdb-dev

matrix:
include:
Expand Down
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ else
USE_SYS_ROCKSDB :=
endif

USE_SYS_ROCKSDB :=
SYS_ROCKSDB := $(if ${USE_SYS_ROCKSDB},ROCKSDB_LIB_DIR=${SYS_LIB_DIR},)

CARGO := env ${SYS_ROCKSDB} cargo
Expand Down
41 changes: 22 additions & 19 deletions built-in-services/asset/src/tests/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,69 +240,72 @@ struct MockStorage;
impl Storage for MockStorage {
async fn insert_transactions(
&self,
_: Context,
_ctx: Context,
_: u64,
_: Vec<SignedTransaction>,
) -> ProtocolResult<()> {
unimplemented!()
}

async fn insert_block(&self, _: Context, _: Block) -> ProtocolResult<()> {
async fn insert_block(&self, _ctx: Context, _: Block) -> ProtocolResult<()> {
unimplemented!()
}

async fn insert_receipts(&self, _: Context, _: Vec<Receipt>) -> ProtocolResult<()> {
async fn insert_receipts(&self, _ctx: Context, _: u64, _: Vec<Receipt>) -> ProtocolResult<()> {
unimplemented!()
}

async fn update_latest_proof(&self, _: Context, _: Proof) -> ProtocolResult<()> {
async fn update_latest_proof(&self, _ctx: Context, _: Proof) -> ProtocolResult<()> {
unimplemented!()
}

async fn get_transaction_by_hash(
&self,
_: Context,
_ctx: Context,
_: Hash,
) -> ProtocolResult<SignedTransaction> {
) -> ProtocolResult<Option<SignedTransaction>> {
unimplemented!()
}

async fn get_transactions(
&self,
_: Context,
_ctx: Context,
_: u64,
_: Vec<Hash>,
) -> ProtocolResult<Vec<SignedTransaction>> {
) -> ProtocolResult<Vec<Option<SignedTransaction>>> {
unimplemented!()
}

async fn get_latest_block(&self, _: Context) -> ProtocolResult<Block> {
async fn get_latest_block(&self, _ctx: Context) -> ProtocolResult<Block> {
unimplemented!()
}

async fn get_block_by_height(&self, _: Context, _: u64) -> ProtocolResult<Block> {
async fn get_block(&self, _ctx: Context, _: u64) -> ProtocolResult<Option<Block>> {
unimplemented!()
}

async fn get_block_by_hash(&self, _: Context, _: Hash) -> ProtocolResult<Block> {
async fn get_receipt_by_hash(&self, _ctx: Context, _: Hash) -> ProtocolResult<Option<Receipt>> {
unimplemented!()
}

async fn get_receipt(&self, _: Context, _: Hash) -> ProtocolResult<Receipt> {
unimplemented!()
}

async fn get_receipts(&self, _: Context, _: Vec<Hash>) -> ProtocolResult<Vec<Receipt>> {
async fn get_receipts(
&self,
_ctx: Context,
_: u64,
_: Vec<Hash>,
) -> ProtocolResult<Vec<Option<Receipt>>> {
unimplemented!()
}

async fn get_latest_proof(&self, _: Context) -> ProtocolResult<Proof> {
async fn get_latest_proof(&self, _ctx: Context) -> ProtocolResult<Proof> {
unimplemented!()
}

async fn update_overlord_wal(&self, _: Context, _info: Bytes) -> ProtocolResult<()> {
async fn update_overlord_wal(&self, _ctx: Context, _info: Bytes) -> ProtocolResult<()> {
unimplemented!()
}

async fn load_overlord_wal(&self, _: Context) -> ProtocolResult<Bytes> {
async fn load_overlord_wal(&self, _ctx: Context) -> ProtocolResult<Bytes> {
unimplemented!()
}
}
41 changes: 22 additions & 19 deletions built-in-services/metadata/src/tests/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,69 +104,72 @@ struct MockStorage;
impl Storage for MockStorage {
async fn insert_transactions(
&self,
_: Context,
_ctx: Context,
_: u64,
_: Vec<SignedTransaction>,
) -> ProtocolResult<()> {
unimplemented!()
}

async fn insert_block(&self, _: Context, _: Block) -> ProtocolResult<()> {
async fn insert_block(&self, _ctx: Context, _: Block) -> ProtocolResult<()> {
unimplemented!()
}

async fn insert_receipts(&self, _: Context, _: Vec<Receipt>) -> ProtocolResult<()> {
async fn insert_receipts(&self, _ctx: Context, _: u64, _: Vec<Receipt>) -> ProtocolResult<()> {
unimplemented!()
}

async fn update_latest_proof(&self, _: Context, _: Proof) -> ProtocolResult<()> {
async fn update_latest_proof(&self, _ctx: Context, _: Proof) -> ProtocolResult<()> {
unimplemented!()
}

async fn get_transaction_by_hash(
&self,
_: Context,
_ctx: Context,
_: Hash,
) -> ProtocolResult<SignedTransaction> {
) -> ProtocolResult<Option<SignedTransaction>> {
unimplemented!()
}

async fn get_transactions(
&self,
_: Context,
_ctx: Context,
_: u64,
_: Vec<Hash>,
) -> ProtocolResult<Vec<SignedTransaction>> {
) -> ProtocolResult<Vec<Option<SignedTransaction>>> {
unimplemented!()
}

async fn get_latest_block(&self, _: Context) -> ProtocolResult<Block> {
async fn get_latest_block(&self, _ctx: Context) -> ProtocolResult<Block> {
unimplemented!()
}

async fn get_block_by_height(&self, _: Context, _: u64) -> ProtocolResult<Block> {
async fn get_block(&self, _ctx: Context, _: u64) -> ProtocolResult<Option<Block>> {
unimplemented!()
}

async fn get_block_by_hash(&self, _: Context, _: Hash) -> ProtocolResult<Block> {
async fn get_receipt_by_hash(&self, _ctx: Context, _: Hash) -> ProtocolResult<Option<Receipt>> {
unimplemented!()
}

async fn get_receipt(&self, _: Context, _: Hash) -> ProtocolResult<Receipt> {
unimplemented!()
}

async fn get_receipts(&self, _: Context, _: Vec<Hash>) -> ProtocolResult<Vec<Receipt>> {
async fn get_receipts(
&self,
_ctx: Context,
_: u64,
_: Vec<Hash>,
) -> ProtocolResult<Vec<Option<Receipt>>> {
unimplemented!()
}

async fn get_latest_proof(&self, _: Context) -> ProtocolResult<Proof> {
async fn get_latest_proof(&self, _ctx: Context) -> ProtocolResult<Proof> {
unimplemented!()
}

async fn update_overlord_wal(&self, _: Context, _info: Bytes) -> ProtocolResult<()> {
async fn update_overlord_wal(&self, _ctx: Context, _info: Bytes) -> ProtocolResult<()> {
unimplemented!()
}

async fn load_overlord_wal(&self, _: Context) -> ProtocolResult<Bytes> {
async fn load_overlord_wal(&self, _ctx: Context) -> ProtocolResult<Bytes> {
unimplemented!()
}
}
28 changes: 18 additions & 10 deletions built-in-services/util/src/tests/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ impl Storage for MockStorage {
async fn insert_transactions(
&self,
_: Context,
_: u64,
_: Vec<SignedTransaction>,
) -> ProtocolResult<()> {
unimplemented!()
Expand All @@ -137,7 +138,12 @@ impl Storage for MockStorage {
unimplemented!()
}

async fn insert_receipts(&self, _: Context, _: Vec<Receipt>) -> ProtocolResult<()> {
async fn insert_receipts(
&self,
_: Context,
_height: u64,
_: Vec<Receipt>,
) -> ProtocolResult<()> {
unimplemented!()
}

Expand All @@ -149,35 +155,37 @@ impl Storage for MockStorage {
&self,
_: Context,
_: Hash,
) -> ProtocolResult<SignedTransaction> {
) -> ProtocolResult<Option<SignedTransaction>> {
unimplemented!()
}

async fn get_transactions(
&self,
_: Context,
_height: u64,
_: Vec<Hash>,
) -> ProtocolResult<Vec<SignedTransaction>> {
) -> ProtocolResult<Vec<Option<SignedTransaction>>> {
unimplemented!()
}

async fn get_latest_block(&self, _: Context) -> ProtocolResult<Block> {
unimplemented!()
}

async fn get_block_by_height(&self, _: Context, _: u64) -> ProtocolResult<Block> {
async fn get_block(&self, _: Context, _: u64) -> ProtocolResult<Option<Block>> {
unimplemented!()
}

async fn get_block_by_hash(&self, _: Context, _: Hash) -> ProtocolResult<Block> {
async fn get_receipt_by_hash(&self, _: Context, _: Hash) -> ProtocolResult<Option<Receipt>> {
unimplemented!()
}

async fn get_receipt(&self, _: Context, _: Hash) -> ProtocolResult<Receipt> {
unimplemented!()
}

async fn get_receipts(&self, _: Context, _: Vec<Hash>) -> ProtocolResult<Vec<Receipt>> {
async fn get_receipts(
&self,
_: Context,
_height: u64,
_: Vec<Hash>,
) -> ProtocolResult<Vec<Option<Receipt>>> {
unimplemented!()
}

Expand Down
38 changes: 28 additions & 10 deletions core/api/src/adapter/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,19 @@ pub enum APIError {
expect
)]
UnExecedError { expect: u64, real: u64 },

#[display(fmt = "not found")]
NotFound,
}

impl std::error::Error for APIError {}

impl From<APIError> for ProtocolError {
fn from(api_err: APIError) -> ProtocolError {
ProtocolError::new(ProtocolErrorKind::API, Box::new(api_err))
}
}

pub struct DefaultAPIAdapter<EF, M, S, DB, Mapping> {
mempool: Arc<M>,
storage: Arc<S>,
Expand Down Expand Up @@ -82,7 +91,11 @@ impl<
height: Option<u64>,
) -> ProtocolResult<Block> {
let block = match height {
Some(id) => self.storage.get_block_by_height(ctx.clone(), id).await?,
Some(id) => self
.storage
.get_block(ctx.clone(), id)
.await?
.ok_or_else(|| APIError::NotFound)?,
None => self.storage.get_latest_block(ctx).await?,
};

Expand All @@ -91,19 +104,21 @@ impl<

#[muta_apm::derive::tracing_span(kind = "API.adapter")]
async fn get_receipt_by_tx_hash(&self, ctx: Context, tx_hash: Hash) -> ProtocolResult<Receipt> {
let receipt = self.storage.get_receipt(ctx.clone(), tx_hash).await?;
let receipt = self
.storage
.get_receipt_by_hash(ctx.clone(), tx_hash)
.await?
.ok_or_else(|| APIError::NotFound)?;
let exec_height = self.storage.get_latest_block(ctx).await?.header.exec_height;
let height = receipt.height;
if exec_height >= height {
return Ok(receipt);
}
Err(ProtocolError::new(
ProtocolErrorKind::API,
Box::new(APIError::UnExecedError {
real: exec_height,
expect: height,
}),
))
Err(APIError::UnExecedError {
real: exec_height,
expect: height,
}
.into())
}

#[muta_apm::derive::tracing_span(kind = "API.adapter")]
Expand All @@ -112,7 +127,10 @@ impl<
ctx: Context,
tx_hash: Hash,
) -> ProtocolResult<SignedTransaction> {
self.storage.get_transaction_by_hash(ctx, tx_hash).await
self.storage
.get_transaction_by_hash(ctx, tx_hash)
.await?
.ok_or_else(|| APIError::NotFound.into())
}

#[muta_apm::derive::tracing_span(kind = "API.adapter")]
Expand Down
Loading

0 comments on commit 49cb7b6

Please sign in to comment.