Skip to content

Commit

Permalink
Add E2E test for indexing from the chain (#122)
Browse files Browse the repository at this point in the history 
## tl;dr

- Adds a `TransactionSigner` interface
- Adds a `GroupMessagePublisher` interface
- Adds the first E2E test that actually publishes to the local blockchain and stores data in the DB
- Renames the `PrivateKeyString` to `SigningKeyString` to differentiate between the message signer and the private key used to sign blockchain transactions
- Adds sqlc to the tools.go to lock in the version
  • Loading branch information
@neekolas
neekolas authored Aug 29, 2024
1 parent 7f252c2 commit b8ab087
Show file tree
Hide file tree
Showing 23 changed files with 384 additions and 49 deletions.
2 changes: 1 addition & 1 deletion .mockery.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ packages:
interfaces:
NodesContract:
NodeRegistry:
github.com/xmtp/xmtpd/pkg/indexer/blockchain:
github.com/xmtp/xmtpd/pkg/blockchain:
interfaces:
ChainClient:
github.com/xmtp/xmtpd/pkg/indexer/storer:
Expand Down
4 changes: 2 additions & 2 deletions cmd/replication/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ func addEnvVars() {
options.DB.ReaderConnectionString = connStr
}

if privKey, hasPrivKey := os.LookupEnv("PRIVATE_KEY"); hasPrivKey {
options.PrivateKeyString = privKey
if privKey, hasPrivKey := os.LookupEnv("SIGNER_PRIVATE_KEY"); hasPrivKey {
options.SignerPrivateKey = privKey
}
}

Expand Down
2 changes: 1 addition & 1 deletion dev/local.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
source dev/contracts/.env

export CHAIN_RPC_URL=$DOCKER_RPC_URL # From contracts/.env
export NODE_PRIVATE_KEY=$PRIVATE_KEY # From contracts/.env
export SIGNER_PRIVATE_KEY=$PRIVATE_KEY # From contracts/.env
export WRITER_CONNECTION_STRING="postgres://postgres:xmtp@localhost:8765/postgres?sslmode=disable"
NODES_CONTRACT_ADDRESS="$(jq -r '.deployedTo' build/Nodes.json)" # Built by contracts/deploy-local
export NODES_CONTRACT_ADDRESS
Expand Down
2 changes: 1 addition & 1 deletion dev/run
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ set -eu

go run cmd/replication/main.go \
--db.writer-connection-string=$WRITER_CONNECTION_STRING \
--private-key=${NODE_PRIVATE_KEY} \
--signer-private-key=${SIGNER_PRIVATE_KEY} \
--contracts.nodes-address=$NODES_CONTRACT_ADDRESS \
--contracts.messages-address=$GROUP_MESSAGES_CONTRACT_ADDRESS \
--contracts.rpc-url=$CHAIN_RPC_URL
2 changes: 1 addition & 1 deletion dev/up
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ if ! which migrate &>/dev/null; then brew install golang-migrate; fi
if ! which golangci-lint &>/dev/null; then brew install golangci-lint; fi
if ! which shellcheck &>/dev/null; then brew install shellcheck; fi
if ! which mockery &>/dev/null; then go install github.com/vektra/mockery/v2; fi
if ! which sqlc &> /dev/null; then brew install sqlc; fi
if ! which sqlc &> /dev/null; then go install github.com/sqlc-dev/sqlc/cmd/sqlc; fi
if ! which buf &> /dev/null; then brew install buf; fi
if ! which golines &>/dev/null; then go install github.com/segmentio/golines@latest; fi
if ! which abigen &>/dev/null; then go install github.com/ethereum/go-ethereum/cmd/abigen; fi
Expand Down
40 changes: 34 additions & 6 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ require (
github.com/ethereum/go-ethereum v1.14.7
github.com/golang-migrate/migrate/v4 v4.17.1
github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0
github.com/jackc/pgx/v5 v5.5.4
github.com/jackc/pgx/v5 v5.6.0
github.com/jessevdk/go-flags v1.6.1
github.com/pires/go-proxyproto v0.7.0
github.com/segmentio/golines v0.12.2
github.com/sqlc-dev/sqlc v1.27.0
github.com/stretchr/testify v1.9.0
github.com/vektra/mockery/v2 v2.44.1
go.uber.org/zap v1.27.0
Expand All @@ -20,6 +21,7 @@ require (
)

require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/DataDog/appsec-internal-go v1.6.0 // indirect
github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect
github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect
Expand All @@ -33,6 +35,7 @@ require (
github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.10.0 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect
Expand All @@ -47,9 +50,11 @@ require (
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/consensys/bavard v0.1.13 // indirect
github.com/consensys/gnark-crypto v0.12.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
github.com/cubicdaiya/gonp v1.0.4 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/dave/dst v0.27.3 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
Expand All @@ -66,17 +71,20 @@ require (
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
github.com/google/cel-go v0.21.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/graph-gophers/graphql-go v1.5.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-bexpr v0.1.10 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
Expand All @@ -93,6 +101,7 @@ require (
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jinzhu/copier v0.3.5 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/klauspost/compress v1.17.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kr/pretty v0.3.1 // indirect
Expand All @@ -109,18 +118,26 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/pointerstructure v1.2.0 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/outcaste-io/ristretto v0.2.3 // indirect
github.com/pelletier/go-toml/v2 v2.0.9 // indirect
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
github.com/pganalyze/pg_query_go/v5 v5.1.0 // indirect
github.com/philhofer/fwd v1.1.2 // indirect
github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 // indirect
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
github.com/pingcap/log v1.1.0 // indirect
github.com/pingcap/tidb/pkg/parser v0.0.0-20231103154709-4f00ece106b1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.12.0 // indirect
github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/riza-io/grpc-go v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/rs/zerolog v1.29.0 // indirect
Expand All @@ -130,30 +147,34 @@ require (
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/afero v1.9.3 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.6.1 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.15.0 // indirect
github.com/status-im/keycard-go v0.2.0 // indirect
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/supranational/blst v0.3.11 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
github.com/tetratelabs/wazero v1.7.3 // indirect
github.com/tinylib/msgp v1.1.8 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
github.com/urfave/cli/v2 v2.25.7 // indirect
github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc // indirect
github.com/wasilibs/wazero-helpers v0.0.0-20240604052452-61d7981e9a38 // indirect
github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
Expand All @@ -165,5 +186,12 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
modernc.org/libc v1.55.3 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.8.0 // indirect
modernc.org/sqlite v1.31.1 // indirect
modernc.org/strutil v1.2.0 // indirect
modernc.org/token v1.1.0 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
)
Loading

0 comments on commit b8ab087

Please sign in to comment.