implement database for scribe (#125)

**Description** Continuing on issue #114 and PR #116 / #118, this PR adds GORM database support for the generic event indexing scribe. It allows for two types of tables to be created, one for Logs and one for Receipts. Both tables have a "store" function as well as a "retrieve" function which is received by txHash.
synapsecns · Aug 24, 2022 · d95be7f · d95be7f
1 parent 9abcce4
commit d95be7f
Show file tree

Hide file tree

Showing 52 changed files with 1,256 additions and 215 deletions.
diff --git a/agents/agents/notary/notary.go b/agents/agents/notary/notary.go
@@ -7,6 +7,7 @@ import (
 	"github.com/synapsecns/sanguine/agents/db/datastore/sql"
 	"github.com/synapsecns/sanguine/agents/domains/evm"
 	"github.com/synapsecns/sanguine/agents/indexer"
+	"github.com/synapsecns/sanguine/core/dbcommon"
 	"github.com/synapsecns/sanguine/ethergo/signer/signer"
 	"golang.org/x/sync/errgroup"
 	"time"
@@ -37,7 +38,7 @@ func NewNotary(ctx context.Context, cfg config.Config) (_ Notary, err error) {
 		return Notary{}, fmt.Errorf("could not create notary: %w", err)
 	}
 
-	dbType, err := sql.DBTypeFromString(cfg.Database.Type)
+	dbType, err := dbcommon.DBTypeFromString(cfg.Database.Type)
 	if err != nil {
 		return Notary{}, fmt.Errorf("could not get legacyDB type: %w", err)
 	}

diff --git a/agents/agents/notary/notary_test.go b/agents/agents/notary/notary_test.go
@@ -8,8 +8,8 @@ import (
 	. "github.com/stretchr/testify/assert"
 	"github.com/synapsecns/sanguine/agents/agents/notary"
 	"github.com/synapsecns/sanguine/agents/config"
-	"github.com/synapsecns/sanguine/agents/db/datastore/sql"
 	"github.com/synapsecns/sanguine/agents/types"
+	"github.com/synapsecns/sanguine/core/dbcommon"
 	"math/big"
 	"time"
 )
@@ -24,7 +24,7 @@ func (u NotarySuite) TestNotaryE2E() {
 			File: filet.TmpFile(u.T(), "", u.wallet.PrivateKeyHex()).Name(),
 		},
 		Database: config.DBConfig{
-			Type:       sql.Sqlite.String(),
+			Type:       dbcommon.Sqlite.String(),
 			DBPath:     filet.TmpDir(u.T(), ""),
 			ConnString: filet.TmpDir(u.T(), ""),
 		},

diff --git a/agents/agents/notary/producer.go b/agents/agents/notary/producer.go
@@ -4,13 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
+
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/synapsecns/sanguine/agents/db"
 	"github.com/synapsecns/sanguine/agents/domains"
 	"github.com/synapsecns/sanguine/agents/types"
 	"github.com/synapsecns/sanguine/core"
 	"github.com/synapsecns/sanguine/ethergo/signer/signer"
-	"time"
 )
 
 // AttestationProducer updates a producer.
@@ -20,7 +21,6 @@ type AttestationProducer struct {
 	domain domains.DomainClient
 	// db is the synapse db
 	db db.SynapseDB
-
 	// signer is the signer
 	signer signer.Signer
 	// interval waits for an interval

diff --git a/agents/agents/notary/submitter.go b/agents/agents/notary/submitter.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
+
 	"github.com/synapsecns/sanguine/agents/db"
 	"github.com/synapsecns/sanguine/agents/domains"
 	"github.com/synapsecns/sanguine/ethergo/signer/signer"
-	"time"
 )
 
 // AttestationSubmitter submits updates continuously.

diff --git a/agents/db/datastore/sql/base/attestation.go b/agents/db/datastore/sql/base/attestation.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/synapsecns/sanguine/agents/db"
+
 	"github.com/synapsecns/sanguine/agents/types"
 	"github.com/synapsecns/sanguine/core"
 	"gorm.io/gorm"
@@ -34,19 +35,19 @@ func (s Store) StoreSignedAttestations(ctx context.Context, attestation types.Si
 	return nil
 }
 
-// RetrieveSignedAttestationByNonce retrieves a signed attesation by nonce.
+// RetrieveSignedAttestationByNonce retrieves a signed attestation by nonce.
 func (s Store) RetrieveSignedAttestationByNonce(ctx context.Context, domainID, nonce uint32) (attestation types.SignedAttestation, err error) {
-	var signedAttesation SignedAttestation
+	var signedAttestation SignedAttestation
 	tx := s.DB().WithContext(ctx).Model(&SignedAttestation{}).Where(&SignedAttestation{
 		SADomain: domainID,
 		SANonce:  nonce,
-	}).First(&signedAttesation)
+	}).First(&signedAttestation)
 
 	if tx.Error != nil {
 		if errors.Is(tx.Error, gorm.ErrRecordNotFound) {
 			return nil, fmt.Errorf("could not find attestation with nonce %d and domain %d: %w", nonce, domainID, db.ErrNotFound)
 		}
 		return nil, fmt.Errorf("could not store attestation: %w", tx.Error)
 	}
-	return signedAttesation, err
+	return signedAttestation, err
 }
diff --git a/agents/db/datastore/sql/base/block.go b/agents/db/datastore/sql/base/block.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/synapsecns/sanguine/agents/db"
+
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 )

diff --git a/agents/db/datastore/sql/base/export_test.go b/agents/db/datastore/sql/base/export_test.go
@@ -9,8 +9,3 @@ import (
 func GetChainID(tx *types.Transaction) (hasType bool, chainID *big.Int) {
 	return getChainID(tx)
 }
-
-// GetGormFieldName wraps getGormFieldName to export it for testing.
-func GetGormFieldName(model interface{}, fieldName string) string {
-	return getGormFieldName(model, fieldName)
-}
diff --git a/agents/db/datastore/sql/base/message.go b/agents/db/datastore/sql/base/message.go
@@ -3,8 +3,9 @@ package base
 import (
 	"context"
 	"fmt"
-	"github.com/Thor-x86/nullable"
 	"github.com/synapsecns/sanguine/agents/db"
+
+	"github.com/Thor-x86/nullable"
 	"github.com/synapsecns/sanguine/agents/types"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm/clause"

diff --git a/agents/db/datastore/sql/base/model.go b/agents/db/datastore/sql/base/model.go
@@ -1,6 +1,7 @@
 package base
 
 import (
+	"github.com/synapsecns/sanguine/core/dbcommon"
 	"math/big"
 	"time"
 
@@ -13,10 +14,11 @@ import (
 // note: some models share names. In cases where they do, we run the check against all names.
 // This is cheap because it's only done at startup.
 func init() {
-	NonceFieldName = getConsistentName("Nonce")
-	DomainIDFieldName = getConsistentName("DomainID")
-	BlockNumberFieldName = getConsistentName("BlockNumber")
-	LeafIndexFieldName = getConsistentName("CMLeafIndex")
+	namer := dbcommon.NewNamer(GetAllModels())
+	NonceFieldName = namer.GetConsistentName("Nonce")
+	DomainIDFieldName = namer.GetConsistentName("DomainID")
+	BlockNumberFieldName = namer.GetConsistentName("BlockNumber")
+	LeafIndexFieldName = namer.GetConsistentName("CMLeafIndex")
 }
 
 var (

diff --git a/agents/db/datastore/sql/base/txqueue.go b/agents/db/datastore/sql/base/txqueue.go
@@ -4,11 +4,11 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"github.com/synapsecns/sanguine/agents/db"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/synapsecns/sanguine/agents/db"
 	"gorm.io/gorm"
 )
 

diff --git a/agents/db/datastore/sql/mysql/logger.go b/agents/db/datastore/sql/mysql/logger.go
@@ -4,4 +4,5 @@ import (
 	"github.com/ipfs/go-log"
 )
 
+// Logger is the mysql logger.
 var logger = log.Logger("synapse-mysql")
diff --git a/agents/db/datastore/sql/mysql/store.go b/agents/db/datastore/sql/mysql/store.go
@@ -3,11 +3,13 @@ package mysql
 import (
 	"context"
 	"fmt"
+	"github.com/synapsecns/sanguine/core/dbcommon"
+	"time"
+
 	"github.com/synapsecns/sanguine/agents/db/datastore/sql/base"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
 	"gorm.io/gorm/schema"
-	"time"
 )
 
 // Store is the sqlite store. It extends the base store for sqlite specific queries.
@@ -26,7 +28,7 @@ func NewMysqlStore(ctx context.Context, dbURL string) (*Store, error) {
 	logger.Debug("creating mysql store")
 
 	gdb, err := gorm.Open(mysql.Open(dbURL), &gorm.Config{
-		Logger:               base.GetGormLogger(logger),
+		Logger:               dbcommon.GetGormLogger(logger),
 		FullSaveAssociations: true,
 		NamingStrategy:       NamingStrategy,
 		NowFunc:              time.Now,

diff --git a/agents/db/datastore/sql/sqlite/logger.go b/agents/db/datastore/sql/sqlite/logger.go
@@ -4,4 +4,5 @@ import (
 	"github.com/ipfs/go-log"
 )
 
+// logger is the sqlite logger.
 var logger = log.Logger("synapse-sqlite")
diff --git a/agents/db/datastore/sql/sqlite/store.go b/agents/db/datastore/sql/sqlite/store.go
@@ -3,10 +3,12 @@ package sqlite
 import (
 	"context"
 	"fmt"
+	common_base "github.com/synapsecns/sanguine/core/dbcommon"
+	"os"
+
 	"github.com/synapsecns/sanguine/agents/db/datastore/sql/base"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
-	"os"
 )
 
 // Store is the sqlite store. It extends the base store for sqlite specific queries.
@@ -26,7 +28,7 @@ func NewSqliteStore(ctx context.Context, dbPath string) (*Store, error) {
 
 	gdb, err := gorm.Open(sqlite.Open(fmt.Sprintf("%s/%s", dbPath, "synapse.db")), &gorm.Config{
 		DisableForeignKeyConstraintWhenMigrating: true,
-		Logger:                                   base.GetGormLogger(logger),
+		Logger:                                   common_base.GetGormLogger(logger),
 		FullSaveAssociations:                     true,
 	})
 	if err != nil {

diff --git a/agents/db/datastore/sql/store.go b/agents/db/datastore/sql/store.go
@@ -3,77 +3,24 @@ package sql
 import (
 	"context"
 	"errors"
-	"fmt"
 	"github.com/synapsecns/sanguine/agents/db"
 	"github.com/synapsecns/sanguine/agents/db/datastore/sql/mysql"
 	"github.com/synapsecns/sanguine/agents/db/datastore/sql/sqlite"
-	"strings"
+	"github.com/synapsecns/sanguine/core/dbcommon"
 )
 
 // NewStoreFromConfig creates a new datastore from a config file.
 // nolint: wrapcheck
-func NewStoreFromConfig(ctx context.Context, dbType DBType, connString string) (db.SynapseDB, error) {
+func NewStoreFromConfig(ctx context.Context, dbType dbcommon.DBType, connString string) (db.SynapseDB, error) {
 	switch dbType {
-	case Mysql:
+	case dbcommon.Mysql:
 		return mysql.NewMysqlStore(ctx, connString)
-	case Sqlite:
+	case dbcommon.Sqlite:
 		return sqlite.NewSqliteStore(ctx, connString)
 	default:
 		return nil, ErrNoSuchDriver
 	}
 }
 
-// DBType is the database driver to use.
-//
-//go:generate go run golang.org/x/tools/cmd/stringer -type=DBType -linecomment
-type DBType int
-
-const (
-	// Mysql is a mysql base db.
-	Mysql DBType = 0 // mysql
-	// Sqlite file based db.
-	Sqlite DBType = iota // sqlite
-)
-
-// DBTypeFromString parses a database type from a string.
-func DBTypeFromString(str string) (DBType, error) {
-	switch strings.ToLower(str) {
-	case Mysql.String():
-		return Mysql, nil
-	case Sqlite.String():
-		return Sqlite, nil
-	default:
-		return DBType(-1), fmt.Errorf("could not convert %s to %T, must be one of %s", str, DBType(-1), allDBTypesList())
-	}
-}
-
-// AllDBTypes is a list of all contract types. Since we use stringer and this is a testing library, instead
-// of manually copying all these out we pull the names out of stringer. In order to make sure stringer is updated, we panic on
-// any method called where the index is higher than the stringer array length.
-var AllDBTypes []DBType
-
-// set all contact types.
-func init() {
-	for i := 0; i < len(_DBType_index)-1; i++ {
-		dbType := DBType(i)
-		AllDBTypes = append(AllDBTypes, dbType)
-
-		// statically assert these are all lowercase
-		if dbType.String() != strings.ToLower(dbType.String()) {
-			panic(fmt.Errorf("db type %s is not lowercase", dbType))
-		}
-	}
-}
-
-// allDBTypesList prints a list of all db types. This is useful for returning errors.
-func allDBTypesList() string {
-	var res []string
-	for _, signerType := range AllDBTypes {
-		res = append(res, signerType.String())
-	}
-
-	return strings.Join(res, ",")
-}
-
 // ErrNoSuchDriver indicates the driver does not exist.
 var ErrNoSuchDriver = errors.New("no such db driver")
diff --git a/agents/go.mod b/agents/go.mod
@@ -10,7 +10,6 @@ require (
 	github.com/brianvoe/gofakeit/v6 v6.9.0
 	github.com/coinbase/rosetta-sdk-go v0.7.0
 	github.com/ethereum/go-ethereum v1.10.11
-	github.com/fatih/structtag v1.2.0
 	github.com/ipfs/go-log v1.0.5
 	github.com/jpillora/backoff v1.0.0
 	github.com/libs4go/crypto v0.0.1
@@ -23,7 +22,6 @@ require (
 	github.com/synapsecns/synapse-node v0.242.0
 	github.com/ugorji/go/codec v1.1.7
 	github.com/vburenin/ifacemaker v1.1.0
-	go.uber.org/zap v1.22.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gorm.io/driver/mysql v1.1.2
 	gorm.io/driver/sqlite v1.1.5
@@ -84,6 +82,7 @@ require (
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/elastic/gosigar v0.10.5 // indirect
 	github.com/fatih/color v1.13.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
@@ -284,6 +283,7 @@ require (
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/zap v1.22.0 // indirect
 	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect
 	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba // indirect

diff --git a/agents/go.sum b/agents/go.sum
@@ -2050,6 +2050,7 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=

diff --git a/agents/indexer/domain.go b/agents/indexer/domain.go
@@ -4,9 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
+
 	"github.com/synapsecns/sanguine/agents/db"
 	"github.com/synapsecns/sanguine/agents/domains"
-	"time"
 )
 
 // domainIndexer indexes a single domain and stores event data in the database.