Merge pull request #19 from cerc-io/roy/v5-dev

* Rewrite trie_by_cid package to be read-write
* add CI workflows

.github/workflows/on-pr.yml

@@ -0,0 +1,7 @@
+name: PR actions
+on:
+  - pull_request
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/test.yml

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Run tests
+on:
+  workflow_call:
+
+jobs:
+  unit-tests:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-go@v4
+      with:
+        go-version-file: 'go.mod'
+        check-latest: true
+    - name: "Run DB container"
+      working-directory: ./test
+      run: |
+        docker compose up -d --quiet-pull
+    - name: "Build and run tests"
+      run: |
+        until [[ "$(docker inspect test-ipld-eth-db | jq -r '.[0].State.Status')" = 'running' ]]
+        do sleep 1; done &
+
+        go build ./...
+        wait $!
+        go test -v ./...

README.md

@@ -1,18 +1,14 @@
 # ipld-eth-statedb
 
-Implementation of the geth [vm.StateDB](https://github.com/ethereum/go-ethereum/blob/master/core/vm/interface.go#L28) on top of
-[ipld-eth-db](https://github.com/cerc-io/ipld-eth-db), to allow us to plug into existing EVM functionality. Analogous to
-[ipfs-ethdb](https://github.com/cerc-io/ipfs-ethdb) but at one database abstraction level higher. This allows us to
-bypass the trie-traversal access pattern normally used by the EVM (and which ipfs-ethdb allows us to replicate ontop of our
-Postgres IPLD blockstore in the "public.blocks" table) and access state and storage directly in the "state_cids" and
-"storage_cids" tables.
+This contains multiple implementations of the geth [vm.StateDB](https://github.com/ethereum/go-ethereum/blob/master/core/vm/interface.go#L28) and supporting types for different use cases.
 
+## Package `direct_by_leaf`
 
-Note: "IPFS" is chosen in the name of "ipfs-ethdb" as it can function through an IPFS BlockService abstraction or directly ontop of an IPLD blockstore, whereas this repository
-is very tightly coupled to the schema in ipld-eth-db.
+A read-only implementation which uses the schema defined in [ipld-eth-db](https://github.com/cerc-io/ipld-eth-db), to allow direct querying by state and storage node leaf key, bypassing the trie-traversal access pattern normally used by the EVM.
+This operates at one abstraction level higher than [ipfs-ethdb](https://github.com/cerc-io/ipfs-ethdb), and is suitable for providing fast state reads.
 
-The top-level package contains the implementation of the `vm.StateDB` interface that accesses state directly using the
-`state_cids` and `storage_cids` tables in ipld-eth-db. The `trie_by_cid` package contains an alternative implementation
-which accesses state in `ipld.blocks` through the typical trie traversal access pattern (using CIDs instead of raw
-keccak256 hashes), it is used for benchmarking and for functionality which requires performing a trie traversal
-(things which must collect intermediate nodes, e.g. `eth_getProof` and `eth_getSlice`).
+## Package `trie_by_cid`
+
+A read-write implementation which uses a Postgres IPLD v0 Blockstore as the backing `ethdb.Database`. Specifically this passes v1 CIDs of Keccak-256 hashes to the database in place of plain hashes, and can be used in combination with a [ipfs-ethdb/postgres/v0](https://github.com/cerc-io/ipfs-ethdb/tree/v5/postgres/v0) `Database` instance, or an IPLD BlockService providing a v0 Blockstore.
+
+This implementation uses trie traversal to access state, and is capable of computing state root hashes and performing full EVM operations. It's also suitable for scenarios requiring trie traversal and access to intermediate state nodes (e.g. `eth_getProof` and `eth_getSlice` on [ipld-eth-server](https://github.com/cerc-io/ipld-eth-server)).

direct_by_leaf/statedb_test.go

@@ -22,7 +22,13 @@ import (
 )
 
 var (
-	testCtx = context.Background()
+	testCtx            = context.Background()
+	teardownStatements = []string{
+		`TRUNCATE eth.header_cids`,
+		`TRUNCATE eth.state_cids`,
+		`TRUNCATE eth.storage_cids`,
+		`TRUNCATE ipld.blocks`,
+	}
 
 	// Fixture data
 	// block one: contract account and slot are created
@@ -96,13 +102,7 @@ func TestPGXSuite(t *testing.T) {
 	t.Cleanup(func() {
 		tx, err := pool.Begin(testCtx)
 		require.NoError(t, err)
-		statements := []string{
-			`TRUNCATE eth.header_cids`,
-			`TRUNCATE eth.state_cids`,
-			`TRUNCATE eth.storage_cids`,
-			`TRUNCATE ipld.blocks`,
-		}
-		for _, stm := range statements {
+		for _, stm := range teardownStatements {
 			_, err = tx.Exec(testCtx, stm)
 			require.NoErrorf(t, err, "Exec(`%s`)", stm)
 		}
@@ -127,13 +127,7 @@ func TestSQLXSuite(t *testing.T) {
 	t.Cleanup(func() {
 		tx, err := pool.Begin()
 		require.NoError(t, err)
-		statements := []string{
-			`TRUNCATE eth.header_cids`,
-			`TRUNCATE eth.state_cids`,
-			`TRUNCATE eth.storage_cids`,
-			`TRUNCATE ipld.blocks`,
-		}
-		for _, stm := range statements {
+		for _, stm := range teardownStatements {
 			_, err = tx.Exec(stm)
 			require.NoErrorf(t, err, "Exec(`%s`)", stm)
 		}

go.mod

@@ -1,10 +1,11 @@
 module github.com/cerc-io/ipld-eth-statedb
 
-go 1.18
+go 1.19
 
 require (
 	github.com/VictoriaMetrics/fastcache v1.6.0
 	github.com/cerc-io/ipfs-ethdb/v5 v5.0.0-alpha
+	github.com/davecgh/go-spew v1.1.1
 	github.com/ethereum/go-ethereum v1.11.5
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
 	github.com/ipfs/go-cid v0.2.0
@@ -13,6 +14,7 @@ require (
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.6
 	github.com/multiformats/go-multihash v0.1.0
+	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.1
 	golang.org/x/crypto v0.6.0
 )
@@ -21,13 +23,13 @@ require (
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
@@ -89,7 +91,6 @@ require (
 	github.com/segmentio/fasthash v1.0.3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/status-im/keycard-go v0.2.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
@@ -112,4 +113,4 @@ require (
 	lukechampine.com/blake3 v1.1.6 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.11.5 => github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.1-alpha
+replace github.com/ethereum/go-ethereum v1.11.5 => github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.3-alpha

go.sum

@@ -17,12 +17,13 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo=
 github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
 github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 h1:KdUfX2zKommPRa+PD0sWZUyXe9w277ABlgELO7H04IM=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.1-alpha h1:x9muuG0Z2W/UAkwAq+0SXhYG9MCP6SJlGEfFNQa6JPI=
-github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.1-alpha/go.mod h1:DIk2wFexjyzvyjuzSOtBEIAPRNZTnLXNbIHEyq1Igek=
+github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.3-alpha h1:rhRmK/NeWMnQ07E4DuLb7WSh9FMotlXMPPaOrf8GJwM=
+github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.3-alpha/go.mod h1:DIk2wFexjyzvyjuzSOtBEIAPRNZTnLXNbIHEyq1Igek=
 github.com/cerc-io/ipfs-ethdb/v5 v5.0.0-alpha h1:I1iXTaIjbTH8ehzNXmT2waXcYBifi1yjK6FK3W3a0Pg=
 github.com/cerc-io/ipfs-ethdb/v5 v5.0.0-alpha/go.mod h1:EGAdV/YewEADFDDVF1k9GNwy8vNWR29Xb87sRHgMIng=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=

internal/util.go

@@ -1,6 +1,10 @@
 package internal
 
 import (
+	"testing"
+	"time"
+
+	pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
 	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 )
@@ -12,3 +16,12 @@ func Keccak256ToCid(codec uint64, h []byte) (cid.Cid, error) {
 	}
 	return cid.NewCidV1(codec, buf), nil
 }
+
+// returns a cache config with unique name (groupcache names are global)
+func MakeCacheConfig(t testing.TB) pgipfsethdb.CacheConfig {
+	return pgipfsethdb.CacheConfig{
+		Name:           t.Name(),
+		Size:           3000000, // 3MB
+		ExpiryDuration: time.Hour,
+	}
+}

test/compose.yml

@@ -0,0 +1,26 @@
+# Containers to run backing DB for unit testing
+
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.2-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "cerc_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
+    container_name: test-ipld-eth-db
+    image: timescale/timescaledb:latest-pg14
+    restart: always
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "cerc_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - "127.0.0.1:8077:5432"

trie_by_cid/doc.go

@@ -0,0 +1,3 @@
+// This package is a near complete copy of go-ethereum/trie and go-ethereum/core/state, modified to use
+// a v0 IPFS blockstore as the backing DB, i.e. DB values are indexed by CID rather than hash.
+package trie_by_cid

trie_by_cid/helper/statediff_helper.go

@@ -20,19 +20,21 @@ var (
 	mockTD = big.NewInt(1)
 )
 
-func IndexChain(dbConfig postgres.Config, stateCache state.Database, rootA, rootB common.Hash) error {
+// IndexStateDiff indexes a single statediff.
+// - uses TestChainConfig
+// - block hash/number are left as zero
+func IndexStateDiff(dbConfig postgres.Config, stateCache state.Database, rootA, rootB common.Hash) error {
 	_, indexer, err := indexer.NewStateDiffIndexer(
 		context.Background(), ChainConfig, node.Info{}, dbConfig)
 	if err != nil {
 		return err
 	}
 	defer indexer.Close() // fixme: hangs when using PGX driver
 
-	// generating statediff payload for block, and transform the data into Postgres
 	builder := statediff.NewBuilder(stateCache)
 	block := types.NewBlock(&types.Header{Root: rootB}, nil, nil, nil, NewHasher())
 
-	// todo: use dummy block hashes to just produce trie structure for testing
+	// uses zero block hash/number, we only need the trie structure here
 	args := statediff.Args{
 		OldStateRoot: rootA,
 		NewStateRoot: rootB,
@@ -45,12 +47,7 @@ func IndexChain(dbConfig postgres.Config, stateCache state.Database, rootA, root
 	if err != nil {
 		return err
 	}
-	// for _, node := range diff.Nodes {
-	// 	err := indexer.PushStateNode(tx, node, block.Hash().String())
-	// 	if err != nil {
-	// 		return err
-	// 	}
-	// }
+	// we don't need to index diff.Nodes since we are just interested in the trie
 	for _, ipld := range diff.IPLDs {
 		if err := indexer.PushIPLD(tx, ipld); err != nil {
 			return err