diff --git a/contrib/promexporter/go.sum b/contrib/promexporter/go.sum index a7e92e37f3..50ac0a109f 100644 --- a/contrib/promexporter/go.sum +++ b/contrib/promexporter/go.sum @@ -117,6 +117,10 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U= +github.com/alecthomas/chroma v0.7.1 h1:G1i02OhUbRi2nJxcNkwJaY/J1gHXj9tt72qN6ZouLFQ= +github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo= +github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkxI1zYWl1QLnEqAqEARBEYa8FQnQcY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -264,6 +268,7 @@ github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbD github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/danielkov/gin-helmet v0.0.0-20171108135313-1387e224435e h1:5jVSh2l/ho6ajWhSPNN84eHEdq3dp0T7+f6r3Tc6hsk= github.com/danielkov/gin-helmet v0.0.0-20171108135313-1387e224435e/go.mod h1:IJgIiGUARc4aOr4bOQ85klmjsShkEEfiRc6q/yBSfo8= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -287,6 +292,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E= github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.23+incompatible h1:1ZQUUYAdh+oylOT85aA2ZcfRp22jmLhoaEcVEfK8dyA= diff --git a/ethergo/backends/mocks/simulated_test_backend.go b/ethergo/backends/mocks/simulated_test_backend.go index add8fd4f68..4affa2206a 100644 --- a/ethergo/backends/mocks/simulated_test_backend.go +++ b/ethergo/backends/mocks/simulated_test_backend.go @@ -312,13 +312,13 @@ func (_m *SimulatedTestBackend) ClientID() string { return r0 } -// CodeAt provides a mock function with given fields: ctx, contract, blockNumber -func (_m *SimulatedTestBackend) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - ret := _m.Called(ctx, contract, blockNumber) +// CodeAt provides a mock function with given fields: ctx, account, blockNumber +func (_m *SimulatedTestBackend) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) { + ret := _m.Called(ctx, account, blockNumber) var r0 []byte if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) []byte); ok { - r0 = rf(ctx, contract, blockNumber) + r0 = rf(ctx, account, blockNumber) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]byte) @@ -327,7 +327,7 @@ func (_m *SimulatedTestBackend) CodeAt(ctx context.Context, contract common.Addr var r1 error if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int) error); ok { - r1 = rf(ctx, contract, blockNumber) + r1 = rf(ctx, account, blockNumber) } else { r1 = ret.Error(1) } diff --git a/ethergo/chain/mocks/chain.go b/ethergo/chain/mocks/chain.go index fcaac507ba..f8dbd2f284 100644 --- a/ethergo/chain/mocks/chain.go +++ b/ethergo/chain/mocks/chain.go @@ -279,13 +279,13 @@ func (_m *Chain) ClientID() string { return r0 } -// CodeAt provides a mock function with given fields: ctx, contract, blockNumber -func (_m *Chain) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - ret := _m.Called(ctx, contract, blockNumber) +// CodeAt provides a mock function with given fields: ctx, account, blockNumber +func (_m *Chain) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) { + ret := _m.Called(ctx, account, blockNumber) var r0 []byte if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) []byte); ok { - r0 = rf(ctx, contract, blockNumber) + r0 = rf(ctx, account, blockNumber) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]byte) @@ -294,7 +294,7 @@ func (_m *Chain) CodeAt(ctx context.Context, contract common.Address, blockNumbe var r1 error if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int) error); ok { - r1 = rf(ctx, contract, blockNumber) + r1 = rf(ctx, account, blockNumber) } else { r1 = ret.Error(1) } diff --git a/services/explorer/api/server_test.go b/services/explorer/api/server_test.go index 1f73f18e6b..0d6e10111d 100644 --- a/services/explorer/api/server_test.go +++ b/services/explorer/api/server_test.go @@ -16,10 +16,9 @@ import ( func TestHandleJSONAmountStat(t *testing.T) { valueString := gofakeit.Word() - // nolint valueStruct := gqlClient.GetAmountStatistic{ Response: &struct { - Value *string `json:"value" graphql:"value"` + Value *string "json:\"value\" graphql:\"value\"" }{ Value: &valueString, }, @@ -31,30 +30,29 @@ func TestHandleJSONAmountStat(t *testing.T) { func TestHandleJSONDailyStat(t *testing.T) { valueFloat := gofakeit.Float64() - // nolint valueStruct := gqlClient.GetDailyStatisticsByChain{ Response: []*struct { - Date *string `json:"date" graphql:"date"` - Ethereum *float64 `json:"ethereum" graphql:"ethereum"` - Optimism *float64 `json:"optimism" graphql:"optimism"` - Cronos *float64 `json:"cronos" graphql:"cronos"` - Bsc *float64 `json:"bsc" graphql:"bsc"` - Polygon *float64 `json:"polygon" graphql:"polygon"` - Fantom *float64 `json:"fantom" graphql:"fantom"` - Boba *float64 `json:"boba" graphql:"boba"` - Metis *float64 `json:"metis" graphql:"metis"` - Moonbeam *float64 `json:"moonbeam" graphql:"moonbeam"` - Moonriver *float64 `json:"moonriver" graphql:"moonriver"` - Klaytn *float64 `json:"klaytn" graphql:"klaytn"` - Arbitrum *float64 `json:"arbitrum" graphql:"arbitrum"` - Avalanche *float64 `json:"avalanche" graphql:"avalanche"` - Dfk *float64 `json:"dfk" graphql:"dfk"` - Aurora *float64 `json:"aurora" graphql:"aurora"` - Harmony *float64 `json:"harmony" graphql:"harmony"` - Canto *float64 `json:"canto" graphql:"canto"` - Dogechain *float64 `json:"dogechain" graphql:"dogechain"` - Base *float64 `json:"base" graphql:"base"` - Total *float64 `json:"total" graphql:"total"` + Date *string "json:\"date\" graphql:\"date\"" + Ethereum *float64 "json:\"ethereum\" graphql:\"ethereum\"" + Optimism *float64 "json:\"optimism\" graphql:\"optimism\"" + Cronos *float64 "json:\"cronos\" graphql:\"cronos\"" + Bsc *float64 "json:\"bsc\" graphql:\"bsc\"" + Polygon *float64 "json:\"polygon\" graphql:\"polygon\"" + Fantom *float64 "json:\"fantom\" graphql:\"fantom\"" + Boba *float64 "json:\"boba\" graphql:\"boba\"" + Metis *float64 "json:\"metis\" graphql:\"metis\"" + Moonbeam *float64 "json:\"moonbeam\" graphql:\"moonbeam\"" + Moonriver *float64 "json:\"moonriver\" graphql:\"moonriver\"" + Klaytn *float64 "json:\"klaytn\" graphql:\"klaytn\"" + Arbitrum *float64 "json:\"arbitrum\" graphql:\"arbitrum\"" + Avalanche *float64 "json:\"avalanche\" graphql:\"avalanche\"" + Dfk *float64 "json:\"dfk\" graphql:\"dfk\"" + Aurora *float64 "json:\"aurora\" graphql:\"aurora\"" + Harmony *float64 "json:\"harmony\" graphql:\"harmony\"" + Canto *float64 "json:\"canto\" graphql:\"canto\"" + Dogechain *float64 "json:\"dogechain\" graphql:\"dogechain\"" + Base *float64 "json:\"base\" graphql:\"base\"" + Total *float64 "json:\"total\" graphql:\"total\"" }{ { Total: &valueFloat, diff --git a/services/explorer/backfill/chain.go b/services/explorer/backfill/chain.go index 42a296f9e5..ece48531ff 100644 --- a/services/explorer/backfill/chain.go +++ b/services/explorer/backfill/chain.go @@ -3,11 +3,9 @@ package backfill import ( "context" "fmt" - "math/big" + "github.com/alecthomas/chroma/lexers/g" "time" - "github.com/synapsecns/sanguine/ethergo/util" - "github.com/ethereum/go-ethereum/common" ethTypes "github.com/ethereum/go-ethereum/core/types" "github.com/jpillora/backoff" @@ -18,8 +16,8 @@ import ( "golang.org/x/sync/errgroup" ) -// ChainBackfiller is an explorer backfiller for a chain. -type ChainBackfiller struct { +// ChainIndexer is an explorer backfiller for a chain. +type ChainIndexer struct { // consumerDB is the database that the backfiller will use to store the events. consumerDB db.ConsumerDB // bridgeParser is the parser to use to parse bridge events. @@ -36,15 +34,17 @@ type ChainBackfiller struct { chainConfig indexerconfig.ChainConfig } +const maxBackoff = 3 + type contextKey string const ( chainKey contextKey = "chainID" ) -// NewChainBackfiller creates a new backfiller for a chain. -func NewChainBackfiller(consumerDB db.ConsumerDB, bridgeParser *parser.BridgeParser, swapParsers map[common.Address]*parser.SwapParser, messageBusParser *parser.MessageBusParser, cctpParser *parser.CCTPParser, fetcher fetcher.ScribeFetcher, chainConfig indexerconfig.ChainConfig) *ChainBackfiller { - return &ChainBackfiller{ +// NewChainIndexer creates a new backfiller for a chain. +func NewChainIndexer(consumerDB db.ConsumerDB, bridgeParser *parser.BridgeParser, swapParsers map[common.Address]*parser.SwapParser, messageBusParser *parser.MessageBusParser, cctpParser *parser.CCTPParser, fetcher fetcher.ScribeFetcher, chainConfig indexerconfig.ChainConfig) *ChainIndexer { + return &ChainIndexer{ consumerDB: consumerDB, bridgeParser: bridgeParser, swapParsers: swapParsers, @@ -55,57 +55,85 @@ func NewChainBackfiller(consumerDB db.ConsumerDB, bridgeParser *parser.BridgePar } } -// Backfill fetches logs from the GraphQL database, parses them, and stores them in the consumer database. +// Index fetches logs from the GraphQL database, parses them, and stores them in the consumer database. // nolint:cyclop,gocognit -func (c *ChainBackfiller) Backfill(ctx context.Context, livefill bool, refreshRate int) (err error) { - chainCtx := context.WithValue(ctx, chainKey, fmt.Sprintf("%d", c.chainConfig.ChainID)) - contractsGroup, contractCtx := errgroup.WithContext(chainCtx) - - if !livefill { - for i := range c.chainConfig.Contracts { - contract := c.chainConfig.Contracts[i] - contractsGroup.Go(func() error { - err := c.backfillContractLogs(contractCtx, contract) - if err != nil { - return fmt.Errorf("could not backfill contract logs: %w", err) - } - return nil - }) +func (c *ChainIndexer) Index(parentContext context.Context, livefill bool, refreshRate int) error { + //chainCtx := context.WithValue(ctx, chainKey, fmt.Sprintf("%d", c.chainConfig.ChainID)) + indexGroup, indexCtx := errgroup.WithContext(parentContext) + + for i := range c.chainConfig.Contracts { + contract := c.chainConfig.Contracts[i] + eventParser, err := c.makeEventParser(contract) + if err != nil { + return fmt.Errorf("invalid contract type", err) } - } else { - for i := range c.chainConfig.Contracts { - contract := c.chainConfig.Contracts[i] - contractsGroup.Go(func() error { - b := &backoff.Backoff{ - Factor: 2, - Jitter: true, - Min: 1 * time.Second, - Max: 3 * time.Second, - } - timeout := time.Duration(0) - for { - select { - case <-chainCtx.Done(): - logger.Errorf("livefill of contract %s on chain %d failed: %v", contract.Address, c.chainConfig.ChainID, chainCtx.Err()) - - return fmt.Errorf("livefill of contract %s on chain %d failed: %w", contract.Address, c.chainConfig.ChainID, chainCtx.Err()) - case <-time.After(timeout): - err := c.backfillContractLogs(contractCtx, contract) - if err != nil { - timeout = b.Duration() - logger.Warnf("could not livefill contract %s on chain %d, retrying %v", contract.Address, c.chainConfig.ChainID, err) - - continue - } - b.Reset() - timeout = time.Duration(refreshRate) * time.Second - logger.Infof("processed range for contract %s on chain %d, continuing to livefill in %d seconds - refresh rate %d ", contract.Address, c.chainConfig.ChainID, timeout, refreshRate) + startHeight := uint64(contract.StartBlock) + + // Spin up a contract indexer + indexGroup.Go(func() error { + + timeout := time.Duration(0) + for { + select { + case <-indexCtx.Done(): + logger.Errorf("livefill of contract %s on chain %d failed: %v", contract.Address, c.chainConfig.ChainID, chainCtx.Err()) + + return fmt.Errorf("livefill of contract %s on chain %d failed: %w", contract.Address, c.chainConfig.ChainID, chainCtx.Err()) + case <-time.After(timeout): + // wrap with retry + err = c.IndexContract(indexCtx, eventParser, startHeight) + if err == nil { + return err + } + if !livefill { + continue } } - }) - } + } + }) } - if err := contractsGroup.Wait(); err != nil { + b := createBackoff() + // + //if !livefill { + // for i := range c.chainConfig.Contracts { + // contract := c.chainConfig.Contracts[i] + // indexGroup.Go(func() error { + // err := c.backfillContractLogs(indexCtx, contract) + // if err != nil { + // return fmt.Errorf("could not backfill contract logs: %w", err) + // } + // return nil + // }) + // } + //} else { + // for i := range c.chainConfig.Contracts { + // contract := c.chainConfig.Contracts[i] + // indexGroup.Go(func() error { + // b := createBackoff() + // timeout := time.Duration(0) + // for { + // select { + // case <-chainCtx.Done(): + // logger.Errorf("livefill of contract %s on chain %d failed: %v", contract.Address, c.chainConfig.ChainID, chainCtx.Err()) + // + // return fmt.Errorf("livefill of contract %s on chain %d failed: %w", contract.Address, c.chainConfig.ChainID, chainCtx.Err()) + // case <-time.After(timeout): + // err := c.backfillContractLogs(indexCtx, contract) + // if err != nil { + // timeout = b.Duration() + // logger.Warnf("could not livefill contract %s on chain %d, retrying %v", contract.Address, c.chainConfig.ChainID, err) + // + // continue + // } + // b.Reset() + // timeout = time.Duration(refreshRate) * time.Second + // logger.Infof("processed range for contract %s on chain %d, continuing to livefill in %d seconds - refresh rate %d ", contract.Address, c.chainConfig.ChainID, timeout, refreshRate) + // } + // } + // }) + // } + //} + if err := indexGroup.Wait(); err != nil { logger.Errorf("error backfilling chain %d completed %v", c.chainConfig.ChainID, err) return fmt.Errorf("error while backfilling chain %d: %w", c.chainConfig.ChainID, err) @@ -115,12 +143,8 @@ func (c *ChainBackfiller) Backfill(ctx context.Context, livefill bool, refreshRa // makeEventParser returns a parser for a contract using it's config. // in the event one is not present, this function will return an error. -func (c *ChainBackfiller) makeEventParser(contract indexerconfig.ContractConfig) (eventParser parser.Parser, err error) { - contractType, err := indexerconfig.ContractTypeFromString(contract.ContractType) - if err != nil { - return nil, fmt.Errorf("could not create event parser for unknown contract type: %s", contract.ContractType) - } - switch contractType { +func (c *ChainIndexer) makeEventParser(contract indexerconfig.ContractConfig) (eventParser parser.Parser, err error) { + switch contract.ContractType { case indexerconfig.BridgeContractType: eventParser = c.bridgeParser case indexerconfig.SwapContractType: @@ -131,20 +155,16 @@ func (c *ChainBackfiller) makeEventParser(contract indexerconfig.ContractConfig) eventParser = c.swapParsers[common.HexToAddress(contract.Address)] case indexerconfig.CCTPContractType: eventParser = c.cctpParser + default: + return nil, fmt.Errorf("could not create event parser for unknown contract type: %s", contract.ContractType) } return eventParser, nil } // backfillContractLogs creates a backfiller for a given contract with an independent context // nolint:cyclop,gocognit -func (c *ChainBackfiller) backfillContractLogs(parentCtx context.Context, contract indexerconfig.ContractConfig) (err error) { +func (c *ChainIndexer) IndexContract(parentCtx context.Context, eventParser parser.Parser, startHeight uint64) (err error) { // make the event parser - eventParser, err := c.makeEventParser(contract) - if err != nil { - return err - } - - startHeight := uint64(contract.StartBlock) // Set start block to -1 to trigger backfill from last block stored by explorer, // otherwise backfilling will begin at the block number specified in the config file. @@ -173,76 +193,38 @@ func (c *ChainBackfiller) backfillContractLogs(parentCtx context.Context, contra // Create context for backfilling chunks g, groupCtx := errgroup.WithContext(parentCtx) - chunkStart := currentHeight - chunkEnd := currentHeight + (c.chainConfig.FetchBlockIncrement-1)*uint64(c.chainConfig.MaxGoroutines) - if chunkEnd > endHeight { - chunkEnd = endHeight + rangeStart := currentHeight + rangeEnd := currentHeight + c.chainConfig.FetchBlockIncrement - 1 + if rangeEnd > endHeight { + rangeEnd = endHeight } + err := c.IndexRange(groupCtx, eventParser, rangeStart, rangeEnd) - iterator := util.NewChunkIterator(big.NewInt(int64(chunkStart)), big.NewInt(int64(chunkEnd)), int(c.chainConfig.FetchBlockIncrement)-1, true) - for subChunk := iterator.NextChunk(); subChunk != nil; subChunk = iterator.NextChunk() { - chunkVar := subChunk - g.Go(func() error { - b := &backoff.Backoff{ - Factor: 2, - Jitter: true, - Min: 1 * time.Second, - Max: 3 * time.Second, - } + //iterator := util.NewChunkIterator(big.NewInt(int64(rangeStart)), big.NewInt(int64(rangeEnd)), int(c.chainConfig.FetchBlockIncrement)-1, true) + //for subChunk := iterator.NextChunk(); subChunk != nil; subChunk = iterator.NextChunk() { + // chunkVar := subChunk + // g.Go(func() error { - timeout := time.Duration(0) - - for { - select { - case <-groupCtx.Done(): - return fmt.Errorf("context canceled: %w", groupCtx.Err()) - case <-time.After(timeout): - rangeEnd := chunkVar.EndBlock.Uint64() - - // Fetch the logs from Scribe. - logs, err := c.Fetcher.FetchLogsInRange(groupCtx, c.chainConfig.ChainID, chunkVar.StartBlock.Uint64(), rangeEnd, common.HexToAddress(contract.Address)) - if err != nil { - timeout = b.Duration() - logger.Warnf("could not fetch logs for chain %d: %v. Retrying in %s", c.chainConfig.ChainID, err, timeout) - - continue - } - - parsedLogs, err := ProcessLogs(groupCtx, logs, c.chainConfig.ChainID, eventParser) - if err != nil { - timeout = b.Duration() - logger.Warnf("could not process logs for chain %d: %s", c.chainConfig.ChainID, err) - continue - } - - if len(parsedLogs) > 0 { - g.Go(func() error { - return c.storeParsedLogs(groupCtx, parsedLogs) - }) - } - return nil - } - } - }) - } + // }) + //} if err := g.Wait(); err != nil { return fmt.Errorf("error while backfilling chain %d: %w", c.chainConfig.ChainID, err) } - logger.Infof("backfilling contract %s chunk completed, %d to %d", contract.Address, chunkStart, chunkEnd) + logger.Infof("backfilling contract %s chunk completed, %d to %d", contract.Address, rangeStart, rangeEnd) // Store the last block in clickhouse err = c.retryWithBackoff(parentCtx, func(ctx context.Context) error { - err = c.consumerDB.StoreLastBlock(parentCtx, c.chainConfig.ChainID, chunkEnd, contract.Address) + err = c.consumerDB.StoreLastBlock(parentCtx, c.chainConfig.ChainID, rangeEnd, contract.Address) if err != nil { return fmt.Errorf("error storing last block, %w", err) } return nil }) if err != nil { - logger.Errorf("could not store last block for chain %d: %s %d, %s, %s", c.chainConfig.ChainID, err, chunkEnd, contract.Address, contract.ContractType) + logger.Errorf("could not store last block for chain %d: %s %d, %s, %s", c.chainConfig.ChainID, err, rangeEnd, contract.Address, contract.ContractType) return fmt.Errorf("could not store last block for chain %d: %w", c.chainConfig.ChainID, err) } - currentHeight = chunkEnd + 1 + currentHeight = rangeEnd + 1 } return nil @@ -258,7 +240,7 @@ func ProcessLogs(ctx context.Context, logs []ethTypes.Log, chainID uint32, event Min: 1 * time.Second, Max: 10 * time.Second, } - + // TODO uint64(c.chainConfig.MaxGoroutines) timeout := time.Duration(0) logIdx := 0 for { @@ -291,7 +273,7 @@ func ProcessLogs(ctx context.Context, logs []ethTypes.Log, chainID uint32, event } } -func (c *ChainBackfiller) storeParsedLogs(ctx context.Context, parsedEvents []interface{}) error { +func (c *ChainIndexer) storeParsedLogs(ctx context.Context, parsedEvents []interface{}) error { b := &backoff.Backoff{ Factor: 2, Jitter: true, @@ -321,7 +303,7 @@ const maxAttempt = 20 type retryableFunc func(ctx context.Context) error // retryWithBackoff will retry to get data with a backoff. -func (c *ChainBackfiller) retryWithBackoff(ctx context.Context, doFunc retryableFunc) error { +func (c *ChainIndexer) retryWithBackoff(ctx context.Context, doFunc retryableFunc) error { b := &backoff.Backoff{ Factor: 2, Jitter: true, @@ -347,3 +329,57 @@ func (c *ChainBackfiller) retryWithBackoff(ctx context.Context, doFunc retryable } return fmt.Errorf("max attempts reached while retrying") } + +func (c *ChainIndexer) IndexRange(parentCtx context.Context, eventParser parser.Parser, rangeStart uint64, rangeEnd uint64) error { + b := &backoff.Backoff{ + Factor: 2, + Jitter: true, + Min: 1 * time.Second, + Max: 3 * time.Second, + } + + timeout := time.Duration(0) + + for { + select { + case <-parentCtx.Done(): + return fmt.Errorf("context canceled: %w", parentCtx.Err()) + case <-time.After(timeout): + //rangeEnd := chunkVar.EndBlock.Uint64() + + // Fetch the logs from Scribe. + logs, err := c.Fetcher.FetchLogsInRange(parentCtx, c.chainConfig.ChainID, rangeStart, rangeEnd, common.HexToAddress(contract.Address)) + if err != nil { + timeout = b.Duration() + logger.Warnf("could not fetch logs for chain %d: %v. Retrying in %s", c.chainConfig.ChainID, err, timeout) + + continue + } + + // TODO add fetch txs and receipts here and then add to the parser. + + parsedLogs, err := ProcessLogs(parentCtx, logs, c.chainConfig.ChainID, eventParser) + if err != nil { + timeout = b.Duration() + logger.Warnf("could not process logs for chain %d: %s", c.chainConfig.ChainID, err) + continue + } + + if len(parsedLogs) > 0 { + g.Go(func() error { + return c.storeParsedLogs(groupCtx, parsedLogs) + }) + } + return nil + } + } +} + +func createBackoff() *backoff.Backoff { + return &backoff.Backoff{ + Factor: 2, + Jitter: true, + Min: 1 * time.Second, + Max: time.Duration(maxBackoff) * time.Second, + } +} diff --git a/services/explorer/backfill/chain_test.go b/services/explorer/backfill/chain_test.go index c1de24167a..17e5ffdb16 100644 --- a/services/explorer/backfill/chain_test.go +++ b/services/explorer/backfill/chain_test.go @@ -3,6 +3,7 @@ package backfill_test import ( gosql "database/sql" "fmt" + "github.com/brianvoe/gofakeit/v6" scribeTypes "github.com/synapsecns/sanguine/services/scribe/types" "math/big" @@ -12,10 +13,8 @@ import ( "github.com/synapsecns/sanguine/services/explorer/static" messageBusTypes "github.com/synapsecns/sanguine/services/explorer/types/messagebus" - "github.com/brianvoe/gofakeit/v6" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - . "github.com/stretchr/testify/assert" "github.com/synapsecns/sanguine/core" "github.com/synapsecns/sanguine/services/explorer/backfill" indexerConfig "github.com/synapsecns/sanguine/services/explorer/config/indexer" @@ -330,7 +329,7 @@ func (b *BackfillSuite) TestBackfill() { Nil(b.T(), err) } - // Set up a ChainBackfiller + // Set up a ChainIndexer bcf, err := fetcher.NewBridgeConfigFetcher(b.bridgeConfigContract.Address(), b.bridgeConfigContract) Nil(b.T(), err) @@ -381,12 +380,12 @@ func (b *BackfillSuite) TestBackfill() { Nil(b.T(), err) // Test the first chain in the config file - chainBackfiller := backfill.NewChainBackfiller(b.db, bp, spMap, mbp, cp, f, chainConfigs[0]) - chainBackfillerV1 := backfill.NewChainBackfiller(b.db, bpv1, spMap, mbp, cp, f, chainConfigsV1[0]) + chainIndexer := backfill.NewChainIndexer(b.db, bp, spMap, mbp, cp, f, chainConfigs[0]) + chainIndexerV1 := backfill.NewChainIndexer(b.db, bpv1, spMap, mbp, cp, f, chainConfigsV1[0]) // Backfill the blocks var count int64 - err = chainBackfiller.Backfill(b.GetTestContext(), false, 1) + err = chainIndexer.Backfill(b.GetTestContext(), false, 1) Nil(b.T(), err) swapEvents := b.db.UNSAFE_DB().WithContext(b.GetTestContext()).Find(&sql.SwapEvent{}).Count(&count) @@ -468,7 +467,7 @@ func (b *BackfillSuite) TestBackfill() { Nil(b.T(), err) // Test bridge v1 parity - err = chainBackfillerV1.Backfill(b.GetTestContext(), false, 1) + err = chainIndexerV1.Backfill(b.GetTestContext(), false, 1) Nil(b.T(), err) err = b.depositParity(depositV1Log, bpv1, uint32(testChainID.Uint64()), true) diff --git a/services/explorer/config/indexer/config.go b/services/explorer/config/indexer/config.go index d7b5d77422..3287814eb7 100644 --- a/services/explorer/config/indexer/config.go +++ b/services/explorer/config/indexer/config.go @@ -13,47 +13,23 @@ import ( "gopkg.in/yaml.v2" ) -// ContractType is the type of contract specified by the config and used for selecting the correct parser. -type ContractType int - +// TODO: these should be put into the contracts themselves and implement a custom type. const ( - // BridgeContractType is the ContractType for the bridge contract. - BridgeContractType ContractType = iota - // SwapContractType is the ContractType for the swap contract. - SwapContractType - // MessageBusContractType is the ContractType for the message bus contract. - MessageBusContractType - // MetaSwapContractType is the ContractType for the meta swap contract. - MetaSwapContractType - // CCTPContractType is the ContractType for the cctp contract. - CCTPContractType + // BridgeContractType is the bridge contract type. + BridgeContractType = "bridge" + // SwapContractType is the swap contract type. + SwapContractType = "swap" + // MessageBusContractType is the message bus contract type. + MessageBusContractType = "messagebus" + // MetaSwapContractType is the meta swap contract type. + MetaSwapContractType = "metaswap" + // CCTPContractType is the CCTP contract type. + CCTPContractType = "cctp" ) -func (c ContractType) String() string { - return [...]string{"bridge", "swap", "messagebus", "metaswap", "cctp"}[c] -} - -// ContractTypeFromString converts a string (intended to be from parsed config) into the ContractType type. -func ContractTypeFromString(s string) (ContractType, error) { - switch s { - case "bridge": - return BridgeContractType, nil - case "swap": - return SwapContractType, nil - case "messagebus": - return MessageBusContractType, nil - case "metaswap": - return MetaSwapContractType, nil - case "cctp": - return CCTPContractType, nil - default: - return -1, fmt.Errorf("unknown contract type: %s", s) - } -} - // Config is used to configure the explorer's data consumption. type Config struct { - // DefaultRefreshRate is the default rate at which data is refreshed. + // DefaultRefreshRate DefaultRefreshRate int `yaml:"default_refresh_rate"` // ScribeURL is the URL of the Scribe server. ScribeURL string `yaml:"scribe_url"` @@ -108,14 +84,14 @@ func (c *Config) IsValid() error { if len(c.BridgeConfigAddress) != (common.AddressLength*2)+2 { return fmt.Errorf("field Address: %w", config.ErrAddressLength) } - if len(c.Chains) == 0 { + if len(c.Chains) > 0 { return fmt.Errorf("no chains specified for indexing") } for _, chain := range c.Chains { err := chain.IsValid() if err != nil { - return fmt.Errorf("chain with ID %d is invalid: %w", chain.ChainID, err) + return err } } @@ -132,12 +108,15 @@ func (c ChainConfig) IsValid() error { case c.MaxGoroutines == 0: return fmt.Errorf("max_goroutines, %w", config.ErrRequiredGlobalField) } + if len(c.Contracts) > 0 { + return fmt.Errorf("no contracts specified for chain %d", c.ChainID) + } intSet := collection.Set[string]{} for _, contract := range c.Contracts { err := contract.IsValid() if err != nil { - return fmt.Errorf("contract with address %s is invalid: %w", contract.Address, err) + return err } if intSet.Contains(contract.Address) { return fmt.Errorf("address %s appears twice", contract.Address) @@ -150,25 +129,19 @@ func (c ChainConfig) IsValid() error { // IsValid validates the chain config. func (c ContractConfig) IsValid() error { - _, err := ContractTypeFromString(c.ContractType) - if err != nil { - return fmt.Errorf("contract_type %s invalid for address %s", c.ContractType, c.Address) - } - switch { case c.StartBlock == 0: return fmt.Errorf("start_block, %w", config.ErrRequiredContractField) case c.Address == "": return fmt.Errorf("address, %w", config.ErrRequiredContractField) + case c.ContractType != BridgeContractType && c.ContractType != SwapContractType && c.ContractType != MessageBusContractType && c.ContractType != MetaSwapContractType && c.ContractType != CCTPContractType: + return fmt.Errorf("contract_type %s invalid for address %s", c.ContractType, c.Address) } return nil } // DecodeConfig parses in a config from a file. func DecodeConfig(filePath string) (cfg Config, err error) { - if _, err := os.Stat(filePath); os.IsNotExist(err) { - return Config{}, fmt.Errorf("config file does not exist: %w", err) - } input, err := os.ReadFile(filepath.Clean(filePath)) if err != nil { return Config{}, fmt.Errorf("failed to read file: %w", err) diff --git a/services/explorer/config/server/config.go b/services/explorer/config/server/config.go index ccd7cba182..cc38201586 100644 --- a/services/explorer/config/server/config.go +++ b/services/explorer/config/server/config.go @@ -73,6 +73,7 @@ func (c *Config) IsValid() error { return fmt.Errorf("db_address, %w", config.ErrRequiredGlobalField) } intSet := collection.Set[uint32]{} + fmt.Println("chains", c.Chains) for _, chain := range c.Chains { err := chain.IsValid() if err != nil { diff --git a/services/explorer/go.mod b/services/explorer/go.mod index a2e3f30554..96d34517e4 100644 --- a/services/explorer/go.mod +++ b/services/explorer/go.mod @@ -1,6 +1,6 @@ module github.com/synapsecns/sanguine/services/explorer -go 1.20 +go 1.19 replace ( github.com/Yamashou/gqlgenc => github.com/synapsecns/gqlgenc v0.10.0-hotfix @@ -20,6 +20,7 @@ require ( github.com/Flaque/filet v0.0.0-20201012163910-45f684403088 github.com/MichaelMure/go-term-markdown v0.1.4 github.com/Yamashou/gqlgenc v0.10.0 + github.com/alecthomas/chroma v0.7.1 github.com/benbjohnson/immutable v0.4.3 github.com/brianvoe/gofakeit/v6 v6.20.1 github.com/ethereum/go-ethereum v1.10.26 @@ -31,7 +32,7 @@ require ( github.com/jftuga/ellipsis v1.0.0 github.com/jftuga/termsize v1.0.2 github.com/jpillora/backoff v1.0.0 - github.com/ory/dockertest/v3 v3.10.0 + github.com/ory/dockertest/v3 v3.9.1 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/ravilushqa/otelgqlgen v0.13.1 github.com/richardwilkes/toolbox v1.74.0 @@ -47,25 +48,31 @@ require ( golang.org/x/sync v0.3.0 gopkg.in/yaml.v2 v2.4.0 gorm.io/driver/clickhouse v0.4.2 - gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 + gorm.io/gorm v1.24.6 k8s.io/apimachinery v0.25.5 ) require ( bitbucket.org/tentontrain/math v0.0.0-20220519191623-a4e86beba92a // indirect - dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/ClickHouse/ch-go v0.47.3 // indirect - github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect + github.com/DataDog/appsec-internal-go v1.0.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.45.0-rc.1 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.45.0 // indirect + github.com/DataDog/datadog-go/v5 v5.1.1 // indirect + github.com/DataDog/go-libddwaf v1.2.0 // indirect + github.com/DataDog/go-tuf v0.3.0--fix-localmeta-fork // indirect + github.com/DataDog/gostackparse v0.5.0 // indirect + github.com/DataDog/sketches-go v1.2.1 // indirect + github.com/ImVexed/fasturl v0.0.0-20200730185836-b0c0fbead04e // indirect github.com/LK4d4/trylock v0.0.0-20191027065348-ff7e133a5c54 // indirect github.com/MichaelMure/go-term-text v0.3.1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect github.com/VictoriaMetrics/fastcache v1.6.0 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect - github.com/alecthomas/chroma v0.7.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/badoux/checkmail v0.0.0-20181210160741-9661bd69e9ad // indirect @@ -81,7 +88,7 @@ require ( github.com/chzyer/logex v1.2.1 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/chzyer/test v1.0.0 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.1.0 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/danielkov/gin-helmet v0.0.0-20171108135313-1387e224435e // indirect @@ -96,6 +103,7 @@ require ( github.com/docker/docker v20.10.23+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -113,9 +121,9 @@ require ( github.com/gin-contrib/zap v0.1.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.6.1 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.4.1 // indirect - github.com/go-git/go-git/v5 v5.8.1 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.4.0 // indirect + github.com/go-git/go-git/v5 v5.5.2 // indirect github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -128,15 +136,15 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.3 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gomarkdown/markdown v0.0.0-20191123064959-2c17d62f5098 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-github/v37 v37.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/pprof v0.0.0-20230509042627-b1315fad0c5a // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/graph-gophers/graphql-go v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect @@ -186,12 +194,12 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/neverlee/keymutex v0.0.0-20171121013845-f593aa834bf9 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/outcaste-io/ristretto v0.2.1 // indirect github.com/palantir/pkg v1.0.1 // indirect github.com/palantir/pkg/datetime v1.0.1 // indirect github.com/palantir/pkg/safejson v1.0.1 // indirect @@ -201,8 +209,9 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/peterh/liner v1.2.1 // indirect + github.com/philhofer/fwd v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect - github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pjbgf/sha1cd v0.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -211,23 +220,26 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/prometheus/tsdb v0.10.0 // indirect - github.com/pyroscope-io/client v0.7.2 // indirect - github.com/pyroscope-io/godeltaprof v0.1.2 // indirect + github.com/pyroscope-io/client v0.7.0 // indirect + github.com/pyroscope-io/godeltaprof v0.1.0 // indirect github.com/pyroscope-io/otel-profiling-go v0.4.0 // indirect + github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rjeczalik/notify v0.9.2 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rs/cors v1.8.2 // indirect github.com/rung/go-safecast v1.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - github.com/skeema/knownhosts v1.2.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/skeema/knownhosts v1.1.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cobra v1.6.1 // indirect @@ -240,6 +252,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/teivah/onecontext v1.3.0 // indirect github.com/tenderly/tenderly-cli v1.4.6 // indirect + github.com/tinylib/msgp v1.1.6 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -268,27 +281,32 @@ require ( go.opentelemetry.io/otel/sdk/metric v0.39.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.25.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.23.0 // indirect + go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.11.0 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/exp v0.0.0-20230127193734-31bee513bff7 // indirect golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/DataDog/dd-trace-go.v1 v1.52.0 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.3.6 // indirect - gorm.io/driver/sqlite v1.5.3 // indirect + gorm.io/driver/sqlite v1.4.3 // indirect + inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect ) diff --git a/services/explorer/graphql/server/graph/fetcher.go b/services/explorer/graphql/server/graph/fetcher.go index 62abe490fd..6946066686 100644 --- a/services/explorer/graphql/server/graph/fetcher.go +++ b/services/explorer/graphql/server/graph/fetcher.go @@ -258,15 +258,13 @@ func (r Resolver) getRangeForHistoricalDestinationLogs(ctx context.Context, chai } timeDifference := int64(blockHeader.Time) - int64(timestamp) - switch { - case -6000 < timeDifference && timeDifference <= 0: + if -6000 < timeDifference && timeDifference <= 0 { return &mid, ¤tBlock, nil - case timeDifference >= 0: + } else if timeDifference >= 0 { upper = mid - default: + } else { lower = mid } - iteration++ } @@ -437,11 +435,15 @@ func (r Resolver) parseSwapLog(ctx context.Context, swapLog ethTypes.Log, chainI if filter == nil { return nil, fmt.Errorf("this swap address is not in the server config, chainid: %d, server: %s", chainID, swapLog.Address.String()) } - iFace, err := filter.ParseTokenSwap(swapLog) - if err != nil || iFace == nil { + swapEvent, err := filter.ParseTokenSwap(swapLog) + if err != nil || swapEvent == nil { return nil, fmt.Errorf("error parsing log, chainid: %d, server: %s", chainID, swapLog.Address.String()) } + iFace, err := filter.ParseTokenSwap(swapLog) + if err != nil { + return nil, fmt.Errorf("could not parse swap event: %w", err) + } soldID := iFace.SoldId address, err := r.DB.GetString(ctx, fmt.Sprintf("SELECT token_address FROM token_indices WHERE contract_address='%s' AND chain_id=%d AND token_index=%d", swapLog.Address.String(), chainID, soldID.Uint64())) if err != nil { @@ -489,9 +491,6 @@ func (r Resolver) storeBridgeEvent(bridgeEvent interface{}) { defer cancel() storeErr := r.DB.StoreEvent(storeCtx, bridgeEvent) if storeErr != nil { - // Log the error and continue. This function is only called by the get origin/destination function, which its only purpose is to return data - // from the chain and return it to the user. If storage fails, it should not disrupt this core purpose. Furthermore, we can assume that in - // the case that storage of this data fails, it will be picked up by scribe and explorer in the next minute and stored correctly. logger.Errorf("could not store log while storing origin bridge watcher tx %v", storeErr) } } diff --git a/services/explorer/graphql/server/graph/queries.resolvers.go b/services/explorer/graphql/server/graph/queries.resolvers.go index c0ee4ef768..7b12221677 100644 --- a/services/explorer/graphql/server/graph/queries.resolvers.go +++ b/services/explorer/graphql/server/graph/queries.resolvers.go @@ -34,24 +34,27 @@ func (r *queryResolver) BridgeTransactions(ctx context.Context, chainIDFrom []*i var toResults []*model.BridgeTransaction var wg sync.WaitGroup - var originErr error - var destinationErr error + var err error wg.Add(1) go func() { defer wg.Done() - fromResults, originErr = r.GetBridgeTxsFromOrigin(ctx, useMv, chainIDFrom, chainIDTo, addressFrom, addressTo, maxAmount, minAmount, maxAmountUsd, minAmountUsd, startTime, endTime, txnHash, tokenAddressTo, tokenAddressFrom, kappa, pending, onlyCctp, page, false) + fromResults, err = r.GetBridgeTxsFromOrigin(ctx, useMv, chainIDFrom, chainIDTo, addressFrom, addressTo, maxAmount, minAmount, maxAmountUsd, minAmountUsd, startTime, endTime, txnHash, tokenAddressTo, tokenAddressFrom, kappa, pending, onlyCctp, page, false) }() wg.Add(1) go func() { defer wg.Done() - toResults, destinationErr = r.GetBridgeTxsFromDestination(ctx, useMv, chainIDFrom, chainIDTo, addressFrom, addressTo, maxAmount, minAmount, minAmountUsd, maxAmountUsd, startTime, endTime, txnHash, kappa, tokenAddressFrom, tokenAddressTo, onlyCctp, page, pending) + toResults, err = r.GetBridgeTxsFromDestination(ctx, useMv, chainIDFrom, chainIDTo, addressFrom, addressTo, maxAmount, minAmount, minAmountUsd, maxAmountUsd, startTime, endTime, txnHash, kappa, tokenAddressFrom, tokenAddressTo, onlyCctp, page, pending) }() wg.Wait() - if originErr != nil || destinationErr != nil { - return nil, fmt.Errorf("error while getting txs. orgin err: %w, destination err: %w", originErr, destinationErr) + if err != nil { + return nil, err } results = r.mergeBridgeTransactions(fromResults, toResults) + + if err != nil { + return nil, fmt.Errorf("failed to get bridge transaction: %w", err) + } sort.Sort(SortBridgeTxType(results)) return results, nil } diff --git a/services/explorer/types/utils.go b/services/explorer/types/utils.go index 50844a9941..6792379ce9 100644 --- a/services/explorer/types/utils.go +++ b/services/explorer/types/utils.go @@ -2,9 +2,14 @@ package types import ( + "github.com/ethereum/go-ethereum/common" + "github.com/synapsecns/sanguine/services/explorer/consumer/fetcher" + "github.com/synapsecns/sanguine/services/explorer/consumer/fetcher/tokenprice" "github.com/synapsecns/sanguine/services/explorer/consumer/parser" + "github.com/synapsecns/sanguine/services/explorer/consumer/parser/tokendata" bridgeContract "github.com/synapsecns/sanguine/services/explorer/contracts/bridge" cctpContract "github.com/synapsecns/sanguine/services/explorer/contracts/cctp" + "github.com/synapsecns/sanguine/services/explorer/db" ) // ServerParsers is a custom type for holding parsers for the server. @@ -18,3 +23,21 @@ type ServerRefs struct { BridgeRefs map[uint32]*bridgeContract.BridgeRef CCTPRefs map[uint32]*cctpContract.CCTPRef } + +// ParserConfig is a custom type for initializing parser. +type ParserConfig struct { + // ContractAddress is the address of the contract. + ContractAddress common.Address + // ConsumerDB is the database to store parsed data in. + ConsumerDB db.ConsumerDB + // CoinGeckoIDs is the mapping of token id to coin gecko ID + CoinGeckoIDs map[string]string + // ConsumerFetcher is the ScribeFetcher for sender and timestamp. + ConsumerFetcher fetcher.ScribeFetcher + // TokenDataService contains the token data service/cache + TokenDataService tokendata.Service + // TokenPriceService contains the token price service/cache + TokenPriceService tokenprice.Service + // FromAPI is true if the parser is being called from the API. + FromAPI bool +} diff --git a/services/scribe/db/datastore/sql/base/log.go b/services/scribe/db/datastore/sql/base/log.go index 9c50ee150c..728d6f84f6 100644 --- a/services/scribe/db/datastore/sql/base/log.go +++ b/services/scribe/db/datastore/sql/base/log.go @@ -252,3 +252,30 @@ func (s Store) retrieveLogsInRangeQuery(ctx context.Context, logFilter db.LogFil return buildLogsFromDBLogs(dbLogs), nil } + +func (s Store) DeleteRangeTemp(ctx context.Context, chainID uint64, startBlock uint64, endBlock uint64) error { + dbTx := s.DB().WithContext(ctx). + Where("chain_id = ?", chainID). + Where("block_number BETWEEN ? AND ?", startBlock, endBlock). + Delete(&Log{}) + if dbTx.Error != nil { + return fmt.Errorf("could not delete logs: %w", dbTx.Error) + } + dbTx = s.DB().WithContext(ctx). + Where("chain_id = ?", chainID). + Where("block_number BETWEEN ? AND ?", startBlock, endBlock). + Delete(&Receipt{}) + if dbTx.Error != nil { + return fmt.Errorf("could not delete rec: %w", dbTx.Error) + } + dbTx = s.DB().WithContext(ctx). + Where("chain_id = ?", chainID). + Where("block_number BETWEEN ? AND ?", startBlock, endBlock). + Delete(&EthTx{}) + + if dbTx.Error != nil { + return fmt.Errorf("could not delete tx: %w", dbTx.Error) + } + + return nil +} diff --git a/services/scribe/db/datastore/sql/mysql/store.go b/services/scribe/db/datastore/sql/mysql/store.go index f45d4dda9a..8dbb276ffa 100644 --- a/services/scribe/db/datastore/sql/mysql/store.go +++ b/services/scribe/db/datastore/sql/mysql/store.go @@ -60,6 +60,7 @@ func NewMysqlStore(parentCtx context.Context, dbURL string, handler metrics.Hand // fixes a timeout issue https://stackoverflow.com/a/42146536 sqlDB.SetMaxIdleConns(MaxIdleConns) sqlDB.SetConnMaxLifetime(30 * time.Minute) + sqlDB.SetConnMaxLifetime(30 * time.Minute) sqlDB.SetMaxOpenConns(MaxOpenConns) handler.AddGormCallbacks(gdb) diff --git a/services/scribe/db/event.go b/services/scribe/db/event.go index 632c8c014f..9c8942eab7 100644 --- a/services/scribe/db/event.go +++ b/services/scribe/db/event.go @@ -102,6 +102,9 @@ type EventDBReader interface { // FlushFromHeadTables flushes unconfirmed logs, receipts, and txs from the head. FlushFromHeadTables(ctx context.Context, time int64) error + + // DeleteRangeTemp + DeleteRangeTemp(ctx context.Context, chainID uint64, startBlock uint64, endBlock uint64) error } // EventDB stores events. diff --git a/services/scribe/db/mocks/event_db.go b/services/scribe/db/mocks/event_db.go index 6404bd40b8..cb08e5c8b1 100644 --- a/services/scribe/db/mocks/event_db.go +++ b/services/scribe/db/mocks/event_db.go @@ -75,6 +75,20 @@ func (_m *EventDB) DeleteLogsForBlockHash(ctx context.Context, blockHash common. return r0 } +// DeleteRangeTemp provides a mock function with given fields: ctx, chainID, startBlock, endBlock +func (_m *EventDB) DeleteRangeTemp(ctx context.Context, chainID uint64, startBlock uint64, endBlock uint64) error { + ret := _m.Called(ctx, chainID, startBlock, endBlock) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, uint64, uint64) error); ok { + r0 = rf(ctx, chainID, startBlock, endBlock) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // DeleteReceiptsForBlockHash provides a mock function with given fields: ctx, chainID, blockHash func (_m *EventDB) DeleteReceiptsForBlockHash(ctx context.Context, chainID uint32, blockHash common.Hash) error { ret := _m.Called(ctx, chainID, blockHash) diff --git a/services/scribe/graphql/client/client.go b/services/scribe/graphql/client/client.go index a02ed9a1f3..3a8473671f 100644 --- a/services/scribe/graphql/client/client.go +++ b/services/scribe/graphql/client/client.go @@ -37,6 +37,7 @@ type Query struct { LogsAtHeadRange []*model.Log "json:\"logsAtHeadRange\" graphql:\"logsAtHeadRange\"" ReceiptsAtHeadRange []*model.Receipt "json:\"receiptsAtHeadRange\" graphql:\"receiptsAtHeadRange\"" TransactionsAtHeadRange []*model.Transaction "json:\"transactionsAtHeadRange\" graphql:\"transactionsAtHeadRange\"" + DelRangeTemp *bool "json:\"delRangeTemp\" graphql:\"delRangeTemp\"" } type GetLogs struct { Response []*struct { @@ -296,6 +297,9 @@ type GetReceiptCount struct { type GetBlockTimeCount struct { Response *int "json:\"response\" graphql:\"response\"" } +type DeleteRangeTemp struct { + Response *bool "json:\"response\" graphql:\"response\"" +} const GetLogsDocument = `query GetLogs ($chain_id: Int!, $page: Int!) { response: logs(chain_id: $chain_id, page: $page) { @@ -899,3 +903,23 @@ func (c *Client) GetBlockTimeCount(ctx context.Context, chainID int, httpRequest return &res, nil } + +const DeleteRangeTempDocument = `query DeleteRangeTemp ($chain_id: Int!, $start_block: Int!, $end_block: Int!) { + response: delRangeTemp(chain_id: $chain_id, start_block: $start_block, end_block: $end_block) +} +` + +func (c *Client) DeleteRangeTemp(ctx context.Context, chainID int, startBlock int, endBlock int, httpRequestOptions ...client.HTTPRequestOption) (*DeleteRangeTemp, error) { + vars := map[string]interface{}{ + "chain_id": chainID, + "start_block": startBlock, + "end_block": endBlock, + } + + var res DeleteRangeTemp + if err := c.Client.Post(ctx, "DeleteRangeTemp", DeleteRangeTempDocument, &res, vars, httpRequestOptions...); err != nil { + return nil, err + } + + return &res, nil +} diff --git a/services/scribe/graphql/client/queries/queries.graphql b/services/scribe/graphql/client/queries/queries.graphql index 19e6238462..9b0a092df5 100644 --- a/services/scribe/graphql/client/queries/queries.graphql +++ b/services/scribe/graphql/client/queries/queries.graphql @@ -283,3 +283,8 @@ query GetReceiptCount ($chain_id: Int!) { query GetBlockTimeCount ($chain_id: Int!) { response: blockTimeCount (chain_id: $chain_id) } + + +query DeleteRangeTemp ($chain_id: Int!, $start_block: Int!, $end_block: Int!) { + response: delRangeTemp (chain_id: $chain_id, start_block: $start_block, end_block: $end_block) +} diff --git a/services/scribe/graphql/server/graph/queries.resolvers.go b/services/scribe/graphql/server/graph/queries.resolvers.go index f4a0b8c6ee..7cd7e0f6f4 100644 --- a/services/scribe/graphql/server/graph/queries.resolvers.go +++ b/services/scribe/graphql/server/graph/queries.resolvers.go @@ -261,6 +261,16 @@ func (r *queryResolver) TransactionsAtHeadRange(ctx context.Context, txHash *str return r.ethTxsToModelTransactions(ctx, transactions, transactionsFilter.ChainID), nil } +// DelRangeTemp is the resolver for the delRangeTemp field. +func (r *queryResolver) DelRangeTemp(ctx context.Context, chainID int, startBlock int, endBlock int) (*bool, error) { + err := r.DB.DeleteRangeTemp(ctx, uint64(chainID), uint64(startBlock), uint64(endBlock)) + if err != nil { + return nil, fmt.Errorf("error deleting range: %w", err) + } + t := true + return &t, nil +} + // Query returns resolvers.QueryResolver implementation. func (r *Resolver) Query() resolvers.QueryResolver { return &queryResolver{r} } diff --git a/services/scribe/graphql/server/graph/resolver/server.go b/services/scribe/graphql/server/graph/resolver/server.go index 05fb0627d9..b2d5898286 100644 --- a/services/scribe/graphql/server/graph/resolver/server.go +++ b/services/scribe/graphql/server/graph/resolver/server.go @@ -73,6 +73,7 @@ type ComplexityRoot struct { Query struct { BlockTime func(childComplexity int, chainID int, blockNumber int) int BlockTimeCount func(childComplexity int, chainID int) int + DelRangeTemp func(childComplexity int, chainID int, startBlock int, endBlock int) int FirstStoredBlockNumber func(childComplexity int, chainID int) int LastConfirmedBlockNumber func(childComplexity int, chainID int) int LastIndexed func(childComplexity int, contractAddress string, chainID int) int @@ -155,6 +156,7 @@ type QueryResolver interface { LogsAtHeadRange(ctx context.Context, contractAddress *string, chainID int, blockNumber *int, txHash *string, txIndex *int, blockHash *string, index *int, confirmed *bool, startBlock int, endBlock int, page int) ([]*model.Log, error) ReceiptsAtHeadRange(ctx context.Context, chainID int, txHash *string, contractAddress *string, blockHash *string, blockNumber *int, txIndex *int, confirmed *bool, startBlock int, endBlock int, page int) ([]*model.Receipt, error) TransactionsAtHeadRange(ctx context.Context, txHash *string, chainID int, blockNumber *int, blockHash *string, confirmed *bool, startBlock int, endBlock int, lastIndexed int, page int) ([]*model.Transaction, error) + DelRangeTemp(ctx context.Context, chainID int, startBlock int, endBlock int) (*bool, error) } type ReceiptResolver interface { Logs(ctx context.Context, obj *model.Receipt) ([]*model.Log, error) @@ -325,6 +327,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.BlockTimeCount(childComplexity, args["chain_id"].(int)), true + case "Query.delRangeTemp": + if e.complexity.Query.DelRangeTemp == nil { + break + } + + args, err := ec.field_Query_delRangeTemp_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.DelRangeTemp(childComplexity, args["chain_id"].(int), args["start_block"].(int), args["end_block"].(int)), true + case "Query.firstStoredBlockNumber": if e.complexity.Query.FirstStoredBlockNumber == nil { break @@ -998,6 +1012,11 @@ directive @goField(forceResolver: Boolean, name: String) on INPUT_FIELD_DEFINITI page: Int! ): [Transaction] + delRangeTemp( + chain_id: Int! + start_block: Int! + end_block: Int! + ): Boolean } `, BuiltIn: false}, @@ -1127,6 +1146,39 @@ func (ec *executionContext) field_Query_blockTime_args(ctx context.Context, rawA return args, nil } +func (ec *executionContext) field_Query_delRangeTemp_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { + var err error + args := map[string]interface{}{} + var arg0 int + if tmp, ok := rawArgs["chain_id"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("chain_id")) + arg0, err = ec.unmarshalNInt2int(ctx, tmp) + if err != nil { + return nil, err + } + } + args["chain_id"] = arg0 + var arg1 int + if tmp, ok := rawArgs["start_block"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("start_block")) + arg1, err = ec.unmarshalNInt2int(ctx, tmp) + if err != nil { + return nil, err + } + } + args["start_block"] = arg1 + var arg2 int + if tmp, ok := rawArgs["end_block"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("end_block")) + arg2, err = ec.unmarshalNInt2int(ctx, tmp) + if err != nil { + return nil, err + } + } + args["end_block"] = arg2 + return args, nil +} + func (ec *executionContext) field_Query_firstStoredBlockNumber_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -4143,6 +4195,58 @@ func (ec *executionContext) fieldContext_Query_transactionsAtHeadRange(ctx conte return fc, nil } +func (ec *executionContext) _Query_delRangeTemp(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_delRangeTemp(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().DelRangeTemp(rctx, fc.Args["chain_id"].(int), fc.Args["start_block"].(int), fc.Args["end_block"].(int)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*bool) + fc.Result = res + return ec.marshalOBoolean2ᚖbool(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_delRangeTemp(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Boolean does not have child fields") + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_delRangeTemp_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return fc, err + } + return fc, nil +} + func (ec *executionContext) _Query___type(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query___type(ctx, field) if err != nil { @@ -8235,6 +8339,25 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) + case "delRangeTemp": + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_delRangeTemp(ctx, field) + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) + } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "__type": out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { diff --git a/services/scribe/graphql/server/graph/schema/queries.graphql b/services/scribe/graphql/server/graph/schema/queries.graphql index aaa237ed3d..02a7beac7d 100644 --- a/services/scribe/graphql/server/graph/schema/queries.graphql +++ b/services/scribe/graphql/server/graph/schema/queries.graphql @@ -149,5 +149,10 @@ type Query { page: Int! ): [Transaction] + delRangeTemp( + chain_id: Int! + start_block: Int! + end_block: Int! + ): Boolean } diff --git a/services/scribe/service/indexer/fetcher.go b/services/scribe/service/indexer/fetcher.go index 6dd69719ed..be9e7da0da 100644 --- a/services/scribe/service/indexer/fetcher.go +++ b/services/scribe/service/indexer/fetcher.go @@ -112,6 +112,7 @@ func (f *LogFetcher) Start(ctx context.Context) error { if err != nil { return fmt.Errorf("could not filter logs: %w", err) } + fmt.Println("fetched logs: ", len(logs), " from ", chunks[0].StartBlock, " to ", chunks[len(chunks)-1].EndBlock, " for ", f.indexerConfig.Addresses) select { case <-ctx.Done(): return fmt.Errorf("context canceled while adding log to chan %w", ctx.Err())