feat: remove RocksDB (#1058)

* refactor: remove RocksDB

* refactor: remove `purge_db`d

* fix: address remove RocksDB PR comments

* feat(db): add "network" column to content table

.circleci/config.yml

@@ -38,7 +38,7 @@ commands:
           name: Install libssl-dev for openssl-sys
           command: sudo apt install -y libssl-dev
       - run:
-          name: Install libclang for rocksdb
+          name: Install libclang
           command: sudo apt install clang
 orbs:
   rust: circleci/[email protected]
@@ -215,7 +215,7 @@ jobs:
           name: Install libssl-dev for openssl-sys
           command: sudo NEEDRESTART_MODE=a apt install -y libssl-dev
       - run:
-          name: Install libclang for rocksdb
+          name: Install libclang
           command: sudo NEEDRESTART_MODE=a apt install clang
       - run:
           name: Install modprobe

Cargo.toml

@@ -27,7 +27,6 @@ prometheus_exporter = "0.8.4"
 rand = "0.8.4"
 reth-ipc = { tag = "v0.1.0-alpha.10", git = "https://github.com/paradigmxyz/reth.git"}
 rlp = "0.5.0"
-rocksdb = "0.21.0"
 rpc = { path = "rpc"}
 serde_json = {version = "1.0.89", features = ["preserve_order"]}
 sha3 = "0.9.1"

book/src/developers/architecture/database.md

@@ -4,30 +4,20 @@ The database related code is located in `./trin-core/src/portalnet/storage.rs`.
 
 There are three main database kinds:
 
-|DB Name|Kind|Location|Purpose|Keys|Values|
-|-|-|-|-|-|-|
-|Main|RocksDB|Disk|Data store|Content ID|Content data bytes|
-|Memory|HashMap|Memory|Kademlia cache|Content key|Content data bytes|
-|Meta|SQLite|Disk|Manage DB size|Content ID|Content key, content size|
+| DB Name |Kind|Location| Purpose        |Keys| Values                                   |
+|---------|-|-|----------------|-|------------------------------------------|
+| Main    |SQLite|Disk| Data store     |Content ID| Content key, content value, content size |
+| Memory  |HashMap|Memory| Kademlia cache |Content key| Content data bytes                       |
 
 ## Main content database
 
-This is a persistent file-based database that uses RocksDB.
-It is also called the "radius" database because content management rules are based on
-the radius of content (specifically the content distance to the node ID).
+This is an SQLite database that stores content data. For a piece of content, this includes
+the content ID, content key, content value and the size of the content. It makes assessing the size of
+the database quicker by avoiding the need to repeatedly compute the size of each content.
 
 ## Memory content database
 
 This uses is an in-memory hashmap to keep content that may not be required for long term
 storage. An overlay service uses this database when receiving data from a peer as
 part of Kademlia-related actions. If required, data is later moved to disk in the
 main content database.
-
-## Meta database
-
-This is an SQLite database that stores metadata. For a piece of content, this includes
-the content ID, content key and the size of the content. It makes assessing the size of
-the main database quicker by avoiding the need to repeatedly compute the size of each content.
-
-Database updates occur in tandum with the main database, where if an operation in one database
-fails, the other can revert the operation to remain synced.

book/src/developers/contributing/build_instructions/linux.md

@@ -14,7 +14,7 @@ These steps are for setting up a Trin node as a service on Ubuntu.
 
 ### Installation
 ```sh
-$ sudo apt install libssl-dev librocksdb-dev libclang-dev pkg-config build-essential
+$ sudo apt install libssl-dev libclang-dev pkg-config build-essential
 ```
 Install Trin:
 > Tip: If you intend to submit code changes to trin, first fork the repo and

book/src/developers/contributing/build_instructions/windows.md

@@ -9,8 +9,7 @@ If you don't already have Rust install it
 $ winget install Rustlang.Rustup
 ```
 
-Install clang/llvm as it is required to compile rocksdb
-If you don't already have Rust install it
+Install clang/llvm as it is required to compile c-kzg
 ```sh
 $ winget install LLVM.LLVM
 ```

book/src/developers/quick_setup.md

@@ -16,7 +16,7 @@ Note: If you use a VPN, you should disable it before running Trin.
 Install dependencies (Ubuntu/Debian):
 
 ```sh
-apt install libssl-dev librocksdb-dev libclang-dev pkg-config build-essential
+apt install libssl-dev libclang-dev pkg-config build-essential
 ```
 
 Environment variables:

docker/Dockerfile

@@ -38,7 +38,6 @@ FROM ubuntu:22.04
 
 # copy build artifacts from build stage
 COPY --from=builder /trin/target/release/trin /usr/bin/
-COPY --from=builder /trin/target/release/purge_db /usr/bin/
 
 ENV RUST_LOG=debug
 

docker/Dockerfile.bridge

@@ -40,7 +40,6 @@ FROM ubuntu:22.04
 # copy build artifacts from build stage
 COPY --from=builder /trin/target/release/trin /usr/bin/
 COPY --from=builder /trin/target/release/portal-bridge /usr/bin/
-COPY --from=builder /trin/target/release/purge_db /usr/bin/
 # These steps copy over the epoch accumulators repo for the bridge to use
 # This data is too large to be kept inside trin-source code
 # It must be downloaded separately and moved to the correct location

ethportal-api/src/types/content_key/history.rs

@@ -61,7 +61,7 @@ impl<'de> Deserialize<'de> for HistoryContentKey {
 }
 
 /// A key for a block header.
-#[derive(Clone, Debug, Decode, Encode, Eq, PartialEq)]
+#[derive(Clone, Debug, Decode, Encode, Eq, PartialEq, Default)]
 pub struct BlockHeaderKey {
     /// Hash of the block.
     pub block_hash: [u8; 32],

ethportal-api/src/types/portal_wire.rs

@@ -219,6 +219,19 @@ impl TryFrom<ProtocolId> for Vec<u8> {
     }
 }
 
+impl From<ProtocolId> for u8 {
+    fn from(protocol_id: ProtocolId) -> Self {
+        match protocol_id {
+            ProtocolId::State => 2,
+            ProtocolId::History => 0,
+            ProtocolId::TransactionGossip => 3,
+            ProtocolId::CanonicalIndices => 4,
+            ProtocolId::Beacon => 1,
+            ProtocolId::Utp => 99,
+        }
+    }
+}
+
 /// A Portal protocol message.
 #[derive(Debug, PartialEq, Clone, Encode, Decode)]
 #[ssz(enum_behaviour = "union")]
@@ -592,6 +605,33 @@ mod test {
         assert_eq!(hex, expected_hex);
     }
 
+    #[test]
+    fn prtocol_id_to_u8() {
+        let protocol_id = ProtocolId::History;
+        let expected_u8 = 0;
+        assert_eq!(expected_u8, u8::from(protocol_id));
+
+        let protocol_id = ProtocolId::Beacon;
+        let expected_u8 = 1;
+        assert_eq!(expected_u8, u8::from(protocol_id));
+
+        let protocol_id = ProtocolId::State;
+        let expected_u8 = 2;
+        assert_eq!(expected_u8, u8::from(protocol_id));
+
+        let protocol_id = ProtocolId::TransactionGossip;
+        let expected_u8 = 3;
+        assert_eq!(expected_u8, u8::from(protocol_id));
+
+        let protocol_id = ProtocolId::CanonicalIndices;
+        let expected_u8 = 4;
+        assert_eq!(expected_u8, u8::from(protocol_id));
+
+        let protocol_id = ProtocolId::Utp;
+        let expected_u8 = 99;
+        assert_eq!(expected_u8, u8::from(protocol_id));
+    }
+
     // Wire message test vectors available in Ethereum Portal Network specs repo:
     // github.com/ethereum/portal-network-specs
 

ethportal-peertest/Cargo.toml

@@ -23,7 +23,6 @@ hyper = { version = "0.14", features = ["full"] }
 jsonrpsee = {version="0.20.0", features = ["async-client", "client", "macros", "server"]}
 rand = "0.8.4"
 reth-ipc = { tag = "v0.1.0-alpha.10", git = "https://github.com/paradigmxyz/reth.git"}
-rocksdb = "0.21.0"
 rpc = { path = "../rpc" }
 serde_json = "1.0.89"
 serde_yaml = "0.9.25"

portalnet/Cargo.toml

@@ -32,7 +32,6 @@ lru = "0.7.8"
 parking_lot = "0.11.2"
 rand = "0.8.4"
 rlp = "0.5.0"
-rocksdb = "0.21.0"
 rusqlite = { version = "0.26.3", features = ["bundled"] }
 r2d2 = "0.8.9"
 r2d2_sqlite = "0.19.0"