From b91174cd64e3b4a825cf58766f9e34a62484a598 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 23 Jan 2025 17:47:07 +0200 Subject: [PATCH 01/17] Index incoming tokens --- cmd/elasticindexer/config/prefs.toml | 7 + config/config.go | 10 +- data/tokens.go | 12 ++ factory/runType/interface.go | 4 +- factory/runType/runTypeComponents.go | 6 +- factory/runType/runTypeComponentsFactory.go | 10 +- factory/runType/runTypeComponentsHandler.go | 21 ++- .../sovereignRunTypeComponentsFactory.go | 28 +++- factory/wsIndexerFactory.go | 9 ++ integrationtests/consts.go | 3 +- integrationtests/incomingSCR_test.go | 99 +++++++++++++ .../testdata/incomingSCR/incoming-scr.json | 38 +++++ integrationtests/utils.go | 34 ++++- process/elasticproc/elasticProcessor.go | 11 +- .../factory/elasticProcessorFactory.go | 11 ++ process/elasticproc/interface.go | 7 + .../elasticproc/tokens/indexTokensHandler.go | 21 +++ .../tokens/sovereignIndexTokensHandler.go | 133 ++++++++++++++++++ process/elasticproc/transactions/errors.go | 3 + process/factory/indexerFactory.go | 5 +- 20 files changed, 450 insertions(+), 22 deletions(-) create mode 100644 integrationtests/incomingSCR_test.go create mode 100644 integrationtests/testdata/incomingSCR/incoming-scr.json create mode 100644 process/elasticproc/tokens/indexTokensHandler.go create mode 100644 process/elasticproc/tokens/sovereignIndexTokensHandler.go diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 2685e4b8..49100748 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -23,3 +23,10 @@ username = "" password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB + + [config.main-elastic-cluster] + use-kibana = false + url = "https://testnet-index.multiversx.com" + username = "" + password = "" + bulk-request-max-size-in-bytes = 4194304 # 4MB diff --git a/config/config.go b/config/config.go index 5d4e7812..6e864804 100644 --- a/config/config.go +++ b/config/config.go @@ -29,7 +29,8 @@ type Config struct { LogsPath string `toml:"logs-path"` } `toml:"logs"` } `toml:"config"` - Sovereign bool + Sovereign bool + ESDTPrefix string } // ClusterConfig will hold the config for the Elasticsearch cluster @@ -52,6 +53,13 @@ type ClusterConfig struct { Password string `toml:"password"` BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` + MainChainCluster struct { + UseKibana bool `toml:"use-kibana"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` + BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + } `toml:"main-elastic-cluster"` } `toml:"config"` } diff --git a/data/tokens.go b/data/tokens.go index e878facf..0c47f651 100644 --- a/data/tokens.go +++ b/data/tokens.go @@ -42,6 +42,18 @@ type SourceToken struct { CurrentOwner string `json:"currentOwner"` } +// ResponseTokenInfo is the structure for the token info response +type ResponseTokenInfo struct { + Docs []ResponseTokenInfoDB `json:"docs"` +} + +// ResponseTokenInfoDB is the structure for the token info response +type ResponseTokenInfoDB struct { + Found bool `json:"found"` + ID string `json:"_id"` + Source TokenInfo `json:"_source"` +} + // TokenInfo is a structure that is needed to store information about a token type TokenInfo struct { Name string `json:"name,omitempty"` diff --git a/factory/runType/interface.go b/factory/runType/interface.go index 1114ff4d..4b96663b 100644 --- a/factory/runType/interface.go +++ b/factory/runType/interface.go @@ -1,12 +1,13 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) // RunTypeComponentsCreator is the interface for creating run type components type RunTypeComponentsCreator interface { - Create() *runTypeComponents + Create() (*runTypeComponents, error) IsInterfaceNil() bool } @@ -28,6 +29,7 @@ type RunTypeComponentsHandler interface { type RunTypeComponentsHolder interface { TxHashExtractorCreator() transactions.TxHashExtractor RewardTxDataCreator() transactions.RewardTxDataHandler + IndexTokensHandlerCreator() elasticproc.IndexTokensHandler Create() error Close() error CheckSubcomponents() error diff --git a/factory/runType/runTypeComponents.go b/factory/runType/runTypeComponents.go index b70512b6..4bccd66b 100644 --- a/factory/runType/runTypeComponents.go +++ b/factory/runType/runTypeComponents.go @@ -1,12 +1,14 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) type runTypeComponents struct { - txHashExtractor transactions.TxHashExtractor - rewardTxData transactions.RewardTxDataHandler + txHashExtractor transactions.TxHashExtractor + rewardTxData transactions.RewardTxDataHandler + indexTokensHandler elasticproc.IndexTokensHandler } // Close does nothing diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go index 8ec94df3..af7d42a3 100644 --- a/factory/runType/runTypeComponentsFactory.go +++ b/factory/runType/runTypeComponentsFactory.go @@ -1,6 +1,7 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) @@ -12,11 +13,12 @@ func NewRunTypeComponentsFactory() *runTypeComponentsFactory { } // Create will create the run type components -func (rtcf *runTypeComponentsFactory) Create() *runTypeComponents { +func (rtcf *runTypeComponentsFactory) Create() (*runTypeComponents, error) { return &runTypeComponents{ - txHashExtractor: transactions.NewTxHashExtractor(), - rewardTxData: transactions.NewRewardTxData(), - } + txHashExtractor: transactions.NewTxHashExtractor(), + rewardTxData: transactions.NewRewardTxData(), + indexTokensHandler: tokens.NewIndexTokensHandler(), + }, nil } // IsInterfaceNil returns true if there is no value under the interface diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 6b0fe9e9..1d471a5c 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) @@ -34,7 +35,10 @@ func NewManagedRunTypeComponents(rtc RunTypeComponentsCreator) (*managedRunTypeC // Create will create the managed components func (mrtc *managedRunTypeComponents) Create() error { - rtc := mrtc.factory.Create() + rtc, err := mrtc.factory.Create() + if err != nil { + return err + } mrtc.mutRunTypeCoreComponents.Lock() mrtc.runTypeComponents = rtc @@ -75,6 +79,9 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { if check.IfNil(mrtc.rewardTxData) { return transactions.ErrNilRewardTxDataHandler } + if check.IfNil(mrtc.indexTokensHandler) { + return transactions.ErrNilIndexTokensHandler + } return nil } @@ -102,6 +109,18 @@ func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardT return mrtc.runTypeComponents.rewardTxData } +// IndexTokensHandlerCreator return index tokens handler +func (mrtc *managedRunTypeComponents) IndexTokensHandlerCreator() elasticproc.IndexTokensHandler { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + return mrtc.runTypeComponents.indexTokensHandler +} + // IsInterfaceNil returns true if the interface is nil func (mrtc *managedRunTypeComponents) IsInterfaceNil() bool { return mrtc == nil diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 874a1b29..53c8d767 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -1,22 +1,36 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) -type sovereignRunTypeComponentsFactory struct{} +type sovereignRunTypeComponentsFactory struct { + mainChainElastic factory.MainChainElastic + esdtPrefix string +} // NewSovereignRunTypeComponentsFactory will return a new instance of sovereign run type components factory -func NewSovereignRunTypeComponentsFactory() *sovereignRunTypeComponentsFactory { - return &sovereignRunTypeComponentsFactory{} +func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.MainChainElastic, esdtPrefix string) *sovereignRunTypeComponentsFactory { + return &sovereignRunTypeComponentsFactory{ + mainChainElastic: mainChainElastic, + esdtPrefix: esdtPrefix, + } } // Create will create the run type components -func (srtcf *sovereignRunTypeComponentsFactory) Create() *runTypeComponents { - return &runTypeComponents{ - txHashExtractor: transactions.NewSovereignTxHashExtractor(), - rewardTxData: transactions.NewSovereignRewardTxData(), +func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, error) { + sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic, srtcf.esdtPrefix) + if err != nil { + return nil, err } + + return &runTypeComponents{ + txHashExtractor: transactions.NewSovereignTxHashExtractor(), + rewardTxData: transactions.NewSovereignRewardTxData(), + indexTokensHandler: sovIndexTokensHandler, + }, nil } // IsInterfaceNil returns true if there is no value under the interface diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index d9d1264c..82d5da64 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/core" + esFactory "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" ) @@ -76,8 +77,16 @@ func createDataIndexer( return nil, err } + mainChainElastic := esFactory.MainChainElastic{ + Url: clusterCfg.Config.MainChainCluster.URL, + UserName: clusterCfg.Config.MainChainCluster.UserName, + Password: clusterCfg.Config.MainChainCluster.Password, + BulkRequestMaxSize: clusterCfg.Config.MainChainCluster.BulkRequestMaxSizeInBytes, + } + return factory.NewIndexer(factory.ArgsIndexerFactory{ Sovereign: cfg.Sovereign, + MainChainElastic: mainChainElastic, UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 8593fcf6..04b25a09 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -4,7 +4,8 @@ const ( //nolint testNumOfShards = 3 //nolint - esURL = "http://localhost:9200" + esURL = "http://localhost:9200" + testnetEsURL = "https://testnet-index.multiversx.com" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go new file mode 100644 index 00000000..cfa6ff3e --- /dev/null +++ b/integrationtests/incomingSCR_test.go @@ -0,0 +1,99 @@ +//go:build integrationtests + +package integrationtests + +import ( + "context" + "encoding/hex" + "math/big" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/stretchr/testify/require" + + indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" +) + +func TestSovereignTransactionWithScCallSuccess(t *testing.T) { + setLogLevelDebug() + + mainChainEs := factory.MainChainElastic{ + Url: testnetEsURL, + } + + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEs) + require.Nil(t, err) + + txHash := []byte("txHash") + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + } + body := &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.SmartContractResultBlock, + SenderShardID: 4294967293, + ReceiverShardID: 0, + TxHashes: [][]byte{txHash}, + }, + }, + } + + token1Identifier := "AGE-be2571" + token2Identifier := "BGD16-c47f46" + data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + + "@02" + + "@" + hex.EncodeToString([]byte(token1Identifier)) + + "@" + + "@" + hex.EncodeToString(big.NewInt(123).Bytes()) + + "@" + hex.EncodeToString([]byte(token2Identifier)) + + "@" + + "@" + hex.EncodeToString(big.NewInt(333).Bytes())) + + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.False(t, token.Found) + } + + pool := &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(txHash): {SmartContractResult: &smartContractResult.SmartContractResult{ + Nonce: 11, + Value: big.NewInt(0), + GasLimit: 0, + SndAddr: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), + RcvAddr: decodeAddress("erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s"), + Data: data, + OriginalTxHash: nil, + }, FeeInfo: &outport.FeeInfo{}}, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, 1)) + require.Nil(t, err) + + ids := []string{hex.EncodeToString(txHash)} + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerData.ScResultsIndex, true, genericResponse) + require.Nil(t, err) + require.JSONEq(t, + readExpectedResult("./testdata/incomingSCR/incoming-scr.json"), + string(genericResponse.Docs[0].Source), + ) + + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json new file mode 100644 index 00000000..09eaf4fb --- /dev/null +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -0,0 +1,38 @@ +{ + "miniBlockHash": "64b954ab72a867bf9866556c7aac3b80a51a69a1d53c640e78f53e5bfcdcc37d", + "nonce": 11, + "gasLimit": 0, + "gasPrice": 0, + "value": "0", + "valueNum": 0, + "sender": "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u", + "receiver": "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", + "senderShard": 4294967293, + "receiverShard": 0, + "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDJANDE0NzQ1MmQ2MjY1MzIzNTM3MzFAQDdiQDQyNDc0NDMxMzYyZDYzMzQzNzY2MzQzNkBAMDE0ZA==", + "prevTxHash": "", + "originalTxHash": "", + "callType": "0", + "timestamp": 5040, + "tokens": [ + "AGE-be2571", + "BGD16-c47f46" + ], + "esdtValues": [ + "123", + "333" + ], + "esdtValuesNum": [ + 1.23e-16, + 3.33e-16 + ], + "receivers": [ + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s" + ], + "receiversShardIDs": [ + 0, + 0 + ], + "operation": "MultiESDTNFTTransfer" +} \ No newline at end of file diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 06239e0c..e2eab567 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -20,12 +20,14 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) var ( log = logger.GetOrCreate("integration-tests") pubKeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, addressPrefix) + sovEsdtPrefix = "sov" ) // nolint @@ -62,9 +64,35 @@ func CreateElasticProcessor( EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, - Denomination: 18, - TxHashExtractor: transactions.NewTxHashExtractor(), - RewardTxData: transactions.NewRewardTxData(), + Denomination: 18, + TxHashExtractor: transactions.NewTxHashExtractor(), + RewardTxData: transactions.NewRewardTxData(), + IndexTokensHandler: tokens.NewIndexTokensHandler(), + } + + return factory.CreateElasticProcessor(args) +} + +// CreateSovereignElasticProcessor - +func CreateSovereignElasticProcessor( + esClient elasticproc.DatabaseClientHandler, + mainChainEs factory.MainChainElastic, +) (dataindexer.ElasticProcessor, error) { + sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainChainEs, sovEsdtPrefix) + + args := factory.ArgElasticProcessorFactory{ + Marshalizer: &mock.MarshalizerMock{}, + Hasher: &mock.HasherMock{}, + AddressPubkeyConverter: pubKeyConverter, + ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32), + DBClient: esClient, + EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, + dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, + dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, + Denomination: 18, + TxHashExtractor: transactions.NewSovereignTxHashExtractor(), + RewardTxData: transactions.NewSovereignRewardTxData(), + IndexTokensHandler: sovIndexTokens, } return factory.CreateElasticProcessor(args) diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 89634a95..e0568504 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -14,6 +14,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -21,7 +23,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" "github.com/multiversx/mx-chain-es-indexer-go/templates" - logger "github.com/multiversx/mx-chain-logger-go" ) var ( @@ -57,6 +58,7 @@ type ArgElasticProcessor struct { LogsAndEventsProc DBLogsAndEventsHandler OperationsProc OperationsHandler Version string + IndexTokensHandler IndexTokensHandler } type elasticProcessor struct { @@ -73,6 +75,7 @@ type elasticProcessor struct { validatorsProc DBValidatorsHandler logsAndEventsProc DBLogsAndEventsHandler operationsProc OperationsHandler + indexTokensHandler IndexTokensHandler } // NewElasticProcessor handles Elasticsearch operations such as initialization, adding, modifying or removing data @@ -94,6 +97,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err logsAndEventsProc: arguments.LogsAndEventsProc, operationsProc: arguments.OperationsProc, bulkRequestMaxSize: arguments.BulkRequestMaxSize, + indexTokensHandler: arguments.IndexTokensHandler, } err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings) @@ -503,6 +507,11 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } + err = ei.indexTokensHandler.IndexCrossChainTokens(ei.elasticClient, preparedResults.ScResults, buffers) + if err != nil { + return err + } + return ei.doBulkRequests("", buffers.Buffers(), obh.ShardID) } diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 54819439..fd361bc9 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -19,6 +19,15 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators" ) +// MainChainElastic holds the elastic search settings +type MainChainElastic struct { + Enabled bool + Url string + UserName string + Password string + BulkRequestMaxSize int +} + // ArgElasticProcessorFactory is struct that is used to store all components that are needed to create an elastic processor factory type ArgElasticProcessorFactory struct { Marshalizer marshal.Marshalizer @@ -34,6 +43,7 @@ type ArgElasticProcessorFactory struct { ImportDB bool TxHashExtractor transactions.TxHashExtractor RewardTxData transactions.RewardTxDataHandler + IndexTokensHandler elasticproc.IndexTokensHandler } // CreateElasticProcessor will create a new instance of ElasticProcessor @@ -132,6 +142,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E OperationsProc: operationsProc, ImportDB: arguments.ImportDB, Version: arguments.Version, + IndexTokensHandler: arguments.IndexTokensHandler, } return elasticproc.NewElasticProcessor(args) diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index fe4b565f..fd5c0af0 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) @@ -120,3 +121,9 @@ type OperationsHandler interface { ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, shardID uint32) ([]*data.Transaction, []*data.ScResult) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string, shardID uint32) error } + +// IndexTokensHandler defines what index tokens handler should be able to do +type IndexTokensHandler interface { + IndexCrossChainTokens(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error + IsInterfaceNil() bool +} diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go new file mode 100644 index 00000000..e48287fd --- /dev/null +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -0,0 +1,21 @@ +package tokens + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" +) + +type indexTokensHandler struct{} + +func NewIndexTokensHandler() *indexTokensHandler { + return &indexTokensHandler{} +} + +func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (it *indexTokensHandler) IsInterfaceNil() bool { + return it == nil +} diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go new file mode 100644 index 00000000..e061ada4 --- /dev/null +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -0,0 +1,133 @@ +package tokens + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "time" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/esdt" + + "github.com/multiversx/mx-chain-es-indexer-go/client" + "github.com/multiversx/mx-chain-es-indexer-go/client/logging" + "github.com/multiversx/mx-chain-es-indexer-go/data" + indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" +) + +type sovereignIndexTokensHandler struct { + mainChainElasticClient elasticproc.DatabaseClientHandler + bulkRequestMaxSize int + esdtPrefix string +} + +func NewSovereignIndexTokensHandler(mainChainElastic factory.MainChainElastic, esdtPrefix string) (*sovereignIndexTokensHandler, error) { + argsEsClient := elasticsearch.Config{ + Addresses: []string{mainChainElastic.Url}, + Username: mainChainElastic.UserName, + Password: mainChainElastic.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: retryBackOff, + } + mainChainElasticClient, err := client.NewElasticClient(argsEsClient) + if err != nil { + return nil, err + } + + return &sovereignIndexTokensHandler{ + mainChainElasticClient: mainChainElasticClient, + bulkRequestMaxSize: mainChainElastic.BulkRequestMaxSize, + esdtPrefix: esdtPrefix, + }, nil +} + +func retryBackOff(attempt int) time.Duration { + return time.Duration(math.Exp2(float64(attempt))) * time.Second +} + +func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + notFoundTokens, err := sit.getTokensFromScrs(elasticClient, scrs) + if err != nil { + return err + } + + if len(notFoundTokens) == 0 { // no new tokens + return nil + } + + // get tokens from main chain elastic db + mainChainTokens := &data.ResponseTokenInfo{} + err = sit.mainChainElasticClient.DoMultiGet(context.Background(), notFoundTokens, indexerdata.TokensIndex, true, mainChainTokens) + if err != nil { + return err + } + + return sit.indexNewTokens(mainChainTokens.Docs, buffSlice) +} + +func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { + receivedTokensIDs := make([]string, 0) + for _, scr := range scrs { + if scr.SenderShard == core.MainChainShardId { + for _, token := range scr.Tokens { + tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) + if !hasPrefix || tokenPrefix != sit.esdtPrefix { + receivedTokensIDs = append(receivedTokensIDs, token) + } + } + } + } + + responseTokens := &data.ResponseTokens{} + err := elasticClient.DoMultiGet(context.Background(), receivedTokensIDs, indexerdata.TokensIndex, true, responseTokens) + if err != nil { + return nil, err + } + + newTokens := make([]string, 0) + for _, token := range responseTokens.Docs { + if token.Found == false { + newTokens = append(newTokens, token.ID) + } + } + + return newTokens, nil +} + +func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { + for _, responseToken := range responseTokensInfo { + token := formatToken(responseToken) + + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, indexerdata.TokensIndex, converters.JsonEscape(token.Token), "\n")) + serializedTokenData, err := json.Marshal(token) + if err != nil { + return err + } + + err = buffSlice.PutData(meta, serializedTokenData) + if err != nil { + return err + } + } + + return nil +} + +func formatToken(token data.ResponseTokenInfoDB) data.TokenInfo { + token.Source.OwnersHistory = nil + token.Source.Properties = nil + + return token.Source +} + +// IsInterfaceNil returns true if there is no value under the interface +func (sit *sovereignIndexTokensHandler) IsInterfaceNil() bool { + return sit == nil +} diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go index 9f91b390..f9280c86 100644 --- a/process/elasticproc/transactions/errors.go +++ b/process/elasticproc/transactions/errors.go @@ -9,3 +9,6 @@ var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") // ErrNilRewardTxDataHandler signals that a nil rewards tx data handler has been provided var ErrNilRewardTxDataHandler = errors.New("nil reward tx data handler") + +// ErrNilIndexTokensHandler signals that a nil index tokens handler has been provided +var ErrNilIndexTokensHandler = errors.New("nil index tokens handler") diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 295039aa..ca18d60d 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -33,6 +33,8 @@ type ArgsIndexerFactory struct { UseKibana bool ImportDB bool Sovereign bool + ESDTPrefix string + MainChainElastic factory.MainChainElastic Denomination int BulkRequestMaxSize int Url string @@ -58,7 +60,7 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { } if args.Sovereign { - args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory()) + args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory(args.MainChainElastic, args.ESDTPrefix)) } else { args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewRunTypeComponentsFactory()) } @@ -126,6 +128,7 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess Version: args.Version, TxHashExtractor: args.RunTypeComponents.TxHashExtractorCreator(), RewardTxData: args.RunTypeComponents.RewardTxDataCreator(), + IndexTokensHandler: args.RunTypeComponents.IndexTokensHandlerCreator(), } return factory.CreateElasticProcessor(argsElasticProcFac) From 706e1fff8d159644fd20ce65cfae7e59c4b5177e Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 23 Jan 2025 17:51:22 +0200 Subject: [PATCH 02/17] removed some unused params --- cmd/elasticindexer/config/prefs.toml | 2 -- config/config.go | 8 +++----- factory/wsIndexerFactory.go | 7 +++---- process/elasticproc/factory/elasticProcessorFactory.go | 9 ++++----- .../elasticproc/tokens/sovereignIndexTokensHandler.go | 2 -- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 49100748..917f006d 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -25,8 +25,6 @@ bulk-request-max-size-in-bytes = 4194304 # 4MB [config.main-elastic-cluster] - use-kibana = false url = "https://testnet-index.multiversx.com" username = "" password = "" - bulk-request-max-size-in-bytes = 4194304 # 4MB diff --git a/config/config.go b/config/config.go index 6e864804..217b5a92 100644 --- a/config/config.go +++ b/config/config.go @@ -54,11 +54,9 @@ type ClusterConfig struct { BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` MainChainCluster struct { - UseKibana bool `toml:"use-kibana"` - URL string `toml:"url"` - UserName string `toml:"username"` - Password string `toml:"password"` - BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` } `toml:"main-elastic-cluster"` } `toml:"config"` } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 82d5da64..1b622e30 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -78,10 +78,9 @@ func createDataIndexer( } mainChainElastic := esFactory.MainChainElastic{ - Url: clusterCfg.Config.MainChainCluster.URL, - UserName: clusterCfg.Config.MainChainCluster.UserName, - Password: clusterCfg.Config.MainChainCluster.Password, - BulkRequestMaxSize: clusterCfg.Config.MainChainCluster.BulkRequestMaxSizeInBytes, + Url: clusterCfg.Config.MainChainCluster.URL, + UserName: clusterCfg.Config.MainChainCluster.UserName, + Password: clusterCfg.Config.MainChainCluster.Password, } return factory.NewIndexer(factory.ArgsIndexerFactory{ diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index fd361bc9..c81053d0 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -21,11 +21,10 @@ import ( // MainChainElastic holds the elastic search settings type MainChainElastic struct { - Enabled bool - Url string - UserName string - Password string - BulkRequestMaxSize int + Enabled bool + Url string + UserName string + Password string } // ArgElasticProcessorFactory is struct that is used to store all components that are needed to create an elastic processor factory diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index e061ada4..b9dd78be 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -23,7 +23,6 @@ import ( type sovereignIndexTokensHandler struct { mainChainElasticClient elasticproc.DatabaseClientHandler - bulkRequestMaxSize int esdtPrefix string } @@ -43,7 +42,6 @@ func NewSovereignIndexTokensHandler(mainChainElastic factory.MainChainElastic, e return &sovereignIndexTokensHandler{ mainChainElasticClient: mainChainElasticClient, - bulkRequestMaxSize: mainChainElastic.BulkRequestMaxSize, esdtPrefix: esdtPrefix, }, nil } From 57b1927831ff77967158ebda498711d2a9268974 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 23 Jan 2025 18:26:26 +0200 Subject: [PATCH 03/17] fixes after testing --- process/elasticproc/tokens/sovereignIndexTokensHandler.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index b9dd78be..f97e4e21 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -83,6 +83,10 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp } } + if len(receivedTokensIDs) == 0 { + return make([]string, 0), nil + } + responseTokens := &data.ResponseTokens{} err := elasticClient.DoMultiGet(context.Background(), receivedTokensIDs, indexerdata.TokensIndex, true, responseTokens) if err != nil { From b5ebb3fb12f4924d2989bf43eca7e60c4b10c618 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 27 Jan 2025 18:11:14 +0200 Subject: [PATCH 04/17] fixes and improvements after self review --- cmd/elasticindexer/config/prefs.toml | 3 +- config/config.go | 1 + docker-compose.yml | 16 ++ .../sovereignRunTypeComponentsFactory.go | 4 +- factory/wsIndexerFactory.go | 3 +- integrationtests/consts.go | 4 +- integrationtests/incomingSCR_test.go | 238 +++++++++++++++--- .../testdata/incomingSCR/incoming-scr.json | 19 +- integrationtests/utils.go | 2 +- .../factory/elasticProcessorFactory.go | 4 +- .../elasticproc/tokens/indexTokensHandler.go | 2 + .../tokens/indexTokensHandler_test.go | 22 ++ .../tokens/sovereignIndexTokensHandler.go | 51 +++- .../sovereignIndexTokensHandler_test.go | 65 +++++ process/factory/indexerFactory.go | 2 +- 15 files changed, 375 insertions(+), 61 deletions(-) create mode 100644 process/elasticproc/tokens/indexTokensHandler_test.go create mode 100644 process/elasticproc/tokens/sovereignIndexTokensHandler_test.go diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 917f006d..7c320e89 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -25,6 +25,7 @@ bulk-request-max-size-in-bytes = 4194304 # 4MB [config.main-elastic-cluster] - url = "https://testnet-index.multiversx.com" + enabled = true + url = "http://localhost:9201" username = "" password = "" diff --git a/config/config.go b/config/config.go index 217b5a92..b5d3b83a 100644 --- a/config/config.go +++ b/config/config.go @@ -54,6 +54,7 @@ type ClusterConfig struct { BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` MainChainCluster struct { + Enabled bool `toml:"enabled"` URL string `toml:"url"` UserName string `toml:"username"` Password string `toml:"password"` diff --git a/docker-compose.yml b/docker-compose.yml index 970f0b40..a5a27b1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,22 @@ services: ports: - "9200:9200" - "9300:9300" + elasticsearch2: + container_name: es-container2 + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 + environment: + - "discovery.type=single-node" + - "xpack.security.enabled=false" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + networks: + - es-net + ports: + - "9201:9200" + - "9301:9300" kibana: container_name: kb-container image: docker.elastic.co/kibana/kibana:7.16.1 diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 53c8d767..2adb96c7 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -7,12 +7,12 @@ import ( ) type sovereignRunTypeComponentsFactory struct { - mainChainElastic factory.MainChainElastic + mainChainElastic factory.ElasticConfig esdtPrefix string } // NewSovereignRunTypeComponentsFactory will return a new instance of sovereign run type components factory -func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.MainChainElastic, esdtPrefix string) *sovereignRunTypeComponentsFactory { +func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.ElasticConfig, esdtPrefix string) *sovereignRunTypeComponentsFactory { return &sovereignRunTypeComponentsFactory{ mainChainElastic: mainChainElastic, esdtPrefix: esdtPrefix, diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 1b622e30..74bc00e4 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -77,7 +77,8 @@ func createDataIndexer( return nil, err } - mainChainElastic := esFactory.MainChainElastic{ + mainChainElastic := esFactory.ElasticConfig{ + Enabled: clusterCfg.Config.MainChainCluster.Enabled, Url: clusterCfg.Config.MainChainCluster.URL, UserName: clusterCfg.Config.MainChainCluster.UserName, Password: clusterCfg.Config.MainChainCluster.Password, diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 04b25a09..9c68a127 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -4,8 +4,8 @@ const ( //nolint testNumOfShards = 3 //nolint - esURL = "http://localhost:9200" - testnetEsURL = "https://testnet-index.multiversx.com" + esURL = "http://localhost:9200" + es2URL = "http://localhost:9201" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index cfa6ff3e..41971e9d 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -5,75 +5,119 @@ package integrationtests import ( "context" "encoding/hex" + "encoding/json" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/stretchr/testify/require" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" ) -func TestSovereignTransactionWithScCallSuccess(t *testing.T) { +type esToken struct { + Identifier string + Value *big.Int + NumDecimals int64 +} + +type esEsdt struct { + Identifier string + Nonce uint64 + EsdtType string + Data esdt.ESDigitalToken +} + +func createTokens() ([]esToken, []esEsdt) { + tokens := []esToken{} + token1 := esToken{ + Identifier: "TKN18-1a2b3c", + Value: big.NewInt(123), + NumDecimals: 18, + } + tokens = append(tokens, token1) + token2 := esToken{ + Identifier: "TKN12-1c2b3a", + Value: big.NewInt(333), + NumDecimals: 12, + } + tokens = append(tokens, token2) + + nfts := []esEsdt{} + nft := esEsdt{ + Identifier: "NFT-abc123", + Nonce: 1, + EsdtType: core.NonFungibleESDTv2, + Data: esdt.ESDigitalToken{ + Type: uint32(core.NonFungibleV2), + Value: big.NewInt(1), + Properties: []byte("3032"), + TokenMetaData: &esdt.MetaData{ + Nonce: 1, + Creator: []byte("creator"), + }, + }, + } + nfts = append(nfts, nft) + + return tokens, nfts +} + +func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() - mainChainEs := factory.MainChainElastic{ - Url: testnetEsURL, + mainChainEs := factory.ElasticConfig{ + Url: es2URL, } + tokens, nfts := createTokens() + createTokensInSourceEs(t, mainChainEs, tokens, nfts) + esClient, err := createESClient(esURL) require.Nil(t, err) esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEs) require.Nil(t, err) - txHash := []byte("txHash") + allTokens := getAllTokensIDs(tokens, nfts) + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.False(t, token.Found) + } + + scrHash := []byte("scrHash") header := &dataBlock.Header{ - Round: 50, - TimeStamp: 5040, + Round: 10, + TimeStamp: 2500, } body := &dataBlock.Body{ MiniBlocks: dataBlock.MiniBlockSlice{ { Type: dataBlock.SmartContractResultBlock, - SenderShardID: 4294967293, - ReceiverShardID: 0, - TxHashes: [][]byte{txHash}, + SenderShardID: core.MainChainShardId, + ReceiverShardID: core.SovereignChainShardId, + TxHashes: [][]byte{scrHash}, }, }, } - token1Identifier := "AGE-be2571" - token2Identifier := "BGD16-c47f46" - data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + - "@02" + - "@" + hex.EncodeToString([]byte(token1Identifier)) + - "@" + - "@" + hex.EncodeToString(big.NewInt(123).Bytes()) + - "@" + hex.EncodeToString([]byte(token2Identifier)) + - "@" + - "@" + hex.EncodeToString(big.NewInt(333).Bytes())) - - genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) - require.Nil(t, err) - for _, token := range genericResponse.Docs { - require.False(t, token.Found) - } - pool := &outport.TransactionPool{ SmartContractResults: map[string]*outport.SCRInfo{ - hex.EncodeToString(txHash): {SmartContractResult: &smartContractResult.SmartContractResult{ + hex.EncodeToString(scrHash): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 11, Value: big.NewInt(0), GasLimit: 0, SndAddr: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeAddress("erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s"), - Data: data, + Data: createMultiEsdtTransferData(tokens, nfts), OriginalTxHash: nil, }, FeeInfo: &outport.FeeInfo{}}, }, @@ -81,9 +125,8 @@ func TestSovereignTransactionWithScCallSuccess(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, 1)) require.Nil(t, err) - ids := []string{hex.EncodeToString(txHash)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(context.Background(), ids, indexerData.ScResultsIndex, true, genericResponse) + err = esClient.DoMultiGet(context.Background(), []string{hex.EncodeToString(scrHash)}, indexerData.ScResultsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/incomingSCR/incoming-scr.json"), @@ -91,9 +134,140 @@ func TestSovereignTransactionWithScCallSuccess(t *testing.T) { ) genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} + +func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esEsdt) { + esClient, err := createESClient(es.Url) + require.Nil(t, err) + + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + body := &dataBlock.Body{} + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + ShardID: core.MetachainShardId, + } + + address1 := "erd1k04pxr6c0gvlcx4rd5fje0a4uy33axqxwz0fpcrgtfdy3nrqauqqgvxprv" + + // create issue token and nft collection events + events := make([]*transaction.Event, 0) + for _, token := range tokens { + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte("issue"), + Topics: [][]byte{[]byte(token.Identifier), []byte("TKN"), []byte("TKN"), []byte(core.FungibleESDT), big.NewInt(token.NumDecimals).Bytes()}, + }) + } + for _, nft := range nfts { + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte("issueNonFungible"), + Topics: [][]byte{[]byte(nft.Identifier), []byte("NFT"), []byte("NFT"), []byte(nft.EsdtType)}, + }) + } + + pool := &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("txHash1")), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: events, + }, + }, + }, + } + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + genericResponse := &GenericResponse{} + allTokens := getAllTokensIDs(tokens, nfts) + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) require.Nil(t, err) for _, token := range genericResponse.Docs { require.True(t, token.Found) } + + // create nft event + events = make([]*transaction.Event, 0) + for _, nft := range nfts { + nftDataBytes, _ := json.Marshal(nft.Data) + + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte(nft.Identifier), big.NewInt(int64(nft.Nonce)).Bytes(), nft.Data.Value.Bytes(), []byte(nftDataBytes)}, + }) + } + + header = &dataBlock.Header{ + Round: 51, + TimeStamp: 5600, + ShardID: 0, + } + + pool = &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("txHash2")), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: events, + }, + }, + }, + } + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + allNfts := make([]string, 0) + for _, nft := range nfts { + allNfts = append(allNfts, nft.Identifier+"-"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())) + } + err = esClient.DoMultiGet(context.Background(), allNfts, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} + +func getAllTokensIDs(tokens []esToken, nfts []esEsdt) []string { + allTokens := make([]string, 0) + for _, token := range tokens { + allTokens = append(allTokens, token.Identifier) + } + for _, nft := range nfts { + allTokens = append(allTokens, nft.Identifier) + } + return allTokens +} + +func createMultiEsdtTransferData(tokens []esToken, nfts []esEsdt) []byte { + data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + + "@" + hex.EncodeToString(big.NewInt(int64(len(tokens)+len(nfts))).Bytes())) + for _, token := range tokens { + data = append(data, []byte( + "@"+hex.EncodeToString([]byte(token.Identifier))+ + "@"+ + "@"+hex.EncodeToString(token.Value.Bytes()))...) + } + for _, nft := range nfts { + nftDataBytes, _ := json.Marshal(nft.Data) + data = append(data, []byte( + "@"+hex.EncodeToString([]byte(nft.Identifier))+ + "@"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())+ + "@"+hex.EncodeToString(nftDataBytes))...) + } + + return data } diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json index 09eaf4fb..29253e9a 100644 --- a/integrationtests/testdata/incomingSCR/incoming-scr.json +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "64b954ab72a867bf9866556c7aac3b80a51a69a1d53c640e78f53e5bfcdcc37d", + "miniBlockHash": "71e255368d7a6686a57a1acb8845953fc54e5a1cfde395acd09df58cb61d5abb", "nonce": 11, "gasLimit": 0, "gasPrice": 0, @@ -9,28 +9,33 @@ "receiver": "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "senderShard": 4294967293, "receiverShard": 0, - "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDJANDE0NzQ1MmQ2MjY1MzIzNTM3MzFAQDdiQDQyNDc0NDMxMzYyZDYzMzQzNzY2MzQzNkBAMDE0ZA==", + "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDNANTQ0YjRlMzEzODJkMzE2MTMyNjIzMzYzQEA3YkA1NDRiNGUzMTMyMmQzMTYzMzI2MjMzNjFAQDAxNGRANGU0NjU0MmQ2MTYyNjMzMTMyMzNAMDFAN2IyMjU0Nzk3MDY1MjIzYTMyMmMyMjU2NjE2Yzc1NjUyMjNhMzEyYzIyNTA3MjZmNzA2NTcyNzQ2OTY1NzMyMjNhMjI0ZDdhNDE3YTRkNjczZDNkMjIyYzIyNGQ2NTc0NjE0NDYxNzQ2MTIyM2E3YjIyNGU2ZjZlNjM2NTIyM2EzMTJjMjI0ZTYxNmQ2NTIyM2E2ZTc1NmM2YzJjMjI0MzcyNjU2MTc0NmY3MjIyM2EyMjU5MzM0YTZjNTk1ODUyNzY2MzY3M2QzZDIyMmMyMjUyNmY3OTYxNmM3NDY5NjU3MzIyM2EzMDJjMjI0ODYxNzM2ODIyM2E2ZTc1NmM2YzJjMjI1NTUyNDk3MzIyM2E2ZTc1NmM2YzJjMjI0MTc0NzQ3MjY5NjI3NTc0NjU3MzIyM2E2ZTc1NmM2YzdkMmMyMjUyNjU3MzY1NzI3NjY1NjQyMjNhNmU3NTZjNmM3ZA==", "prevTxHash": "", "originalTxHash": "", "callType": "0", - "timestamp": 5040, + "timestamp": 2500, "tokens": [ - "AGE-be2571", - "BGD16-c47f46" + "TKN18-1a2b3c", + "TKN12-1c2b3a", + "NFT-abc123-01" ], "esdtValues": [ "123", - "333" + "333", + "1" ], "esdtValuesNum": [ 1.23e-16, - 3.33e-16 + 3.33e-16, + 1e-18 ], "receivers": [ + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s" ], "receiversShardIDs": [ + 0, 0, 0 ], diff --git a/integrationtests/utils.go b/integrationtests/utils.go index e2eab567..0577ca81 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -76,7 +76,7 @@ func CreateElasticProcessor( // CreateSovereignElasticProcessor - func CreateSovereignElasticProcessor( esClient elasticproc.DatabaseClientHandler, - mainChainEs factory.MainChainElastic, + mainChainEs factory.ElasticConfig, ) (dataindexer.ElasticProcessor, error) { sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainChainEs, sovEsdtPrefix) diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index c81053d0..2471885f 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -19,8 +19,8 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators" ) -// MainChainElastic holds the elastic search settings -type MainChainElastic struct { +// ElasticConfig holds the elastic search settings +type ElasticConfig struct { Enabled bool Url string UserName string diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go index e48287fd..d8d78e7c 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -7,10 +7,12 @@ import ( type indexTokensHandler struct{} +// NewIndexTokensHandler creates a new index tokens handler func NewIndexTokensHandler() *indexTokensHandler { return &indexTokensHandler{} } +// IndexCrossChainTokens returns no error func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { return nil } diff --git a/process/elasticproc/tokens/indexTokensHandler_test.go b/process/elasticproc/tokens/indexTokensHandler_test.go new file mode 100644 index 00000000..260c39af --- /dev/null +++ b/process/elasticproc/tokens/indexTokensHandler_test.go @@ -0,0 +1,22 @@ +package tokens + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewIndexTokensHandler(t *testing.T) { + t.Parallel() + + ith := NewIndexTokensHandler() + require.False(t, ith.IsInterfaceNil()) +} + +func TestIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { + t.Parallel() + + ith := NewIndexTokensHandler() + err := ith.IndexCrossChainTokens(nil, nil, nil) + require.NoError(t, err) +} diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index f97e4e21..af9ebea0 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "net/http" + "strings" "time" "github.com/elastic/go-elasticsearch/v7" @@ -22,25 +23,32 @@ import ( ) type sovereignIndexTokensHandler struct { + indexingEnabled bool mainChainElasticClient elasticproc.DatabaseClientHandler esdtPrefix string } -func NewSovereignIndexTokensHandler(mainChainElastic factory.MainChainElastic, esdtPrefix string) (*sovereignIndexTokensHandler, error) { - argsEsClient := elasticsearch.Config{ - Addresses: []string{mainChainElastic.Url}, - Username: mainChainElastic.UserName, - Password: mainChainElastic.Password, - Logger: &logging.CustomLogger{}, - RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, - } - mainChainElasticClient, err := client.NewElasticClient(argsEsClient) - if err != nil { - return nil, err +// NewSovereignIndexTokensHandler creates a new sovereign index tokens handler +func NewSovereignIndexTokensHandler(mainChainElastic factory.ElasticConfig, esdtPrefix string) (*sovereignIndexTokensHandler, error) { + var mainChainElasticClient elasticproc.DatabaseClientHandler + if mainChainElastic.Enabled { + var err error + argsEsClient := elasticsearch.Config{ + Addresses: []string{mainChainElastic.Url}, + Username: mainChainElastic.UserName, + Password: mainChainElastic.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: retryBackOff, + } + mainChainElasticClient, err = client.NewElasticClient(argsEsClient) + if err != nil { + return nil, err + } } return &sovereignIndexTokensHandler{ + indexingEnabled: mainChainElastic.Enabled, mainChainElasticClient: mainChainElasticClient, esdtPrefix: esdtPrefix, }, nil @@ -50,7 +58,12 @@ func retryBackOff(attempt int) time.Duration { return time.Duration(math.Exp2(float64(attempt))) * time.Second } +// IndexCrossChainTokens will index the new tokens properties func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + if !sit.indexingEnabled { + return nil + } + notFoundTokens, err := sit.getTokensFromScrs(elasticClient, scrs) if err != nil { return err @@ -79,6 +92,9 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp if !hasPrefix || tokenPrefix != sit.esdtPrefix { receivedTokensIDs = append(receivedTokensIDs, token) } + if isEsdt, tokenCollection := getTokenCollection(hasPrefix, token); isEsdt { + receivedTokensIDs = append(receivedTokensIDs, tokenCollection) + } } } } @@ -103,6 +119,17 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp return newTokens, nil } +func getTokenCollection(hasPrefix bool, tokenIdentifier string) (bool, string) { + tokenSplit := strings.Split(tokenIdentifier, "-") + if !hasPrefix && len(tokenSplit) == 3 { + return true, tokenSplit[0] + "-" + tokenSplit[1] + } + if hasPrefix && len(tokenSplit) == 4 { + return true, tokenSplit[1] + "-" + tokenSplit[2] + } + return false, "" +} + func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { for _, responseToken := range responseTokensInfo { token := formatToken(responseToken) diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go new file mode 100644 index 00000000..3aebe667 --- /dev/null +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go @@ -0,0 +1,65 @@ +package tokens + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" +) + +const ( + prefix = "sov" +) + +func createElasticConfig() factory.ElasticConfig { + return factory.ElasticConfig{ + Enabled: true, + Url: "http://localhost:9200", + UserName: "", + Password: "", + } +} + +func TestSovereignNewIndexTokensHandler(t *testing.T) { + t.Parallel() + + t.Run("no url, should error", func(t *testing.T) { + esConfig := createElasticConfig() + esConfig.Url = "http://bad url" + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.ErrorContains(t, err, "cannot parse url") + require.Nil(t, sith) + }) + t.Run("not enabled should not create main chain elastic client", func(t *testing.T) { + esConfig := createElasticConfig() + esConfig.Enabled = false + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.NoError(t, err) + require.NotNil(t, sith) + require.Nil(t, sith.mainChainElasticClient) + }) + t.Run("valid config, should work", func(t *testing.T) { + esConfig := createElasticConfig() + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.NoError(t, err) + require.NotNil(t, sith) + }) +} + +func TestSovereignIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { + t.Parallel() + + esConfig := createElasticConfig() + esConfig.Enabled = false + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.NoError(t, err) + require.NotNil(t, sith) + + // should skip indexing + err = sith.IndexCrossChainTokens(nil, make([]*data.ScResult, 0), data.NewBufferSlice(0)) + require.NoError(t, err) + + // actual indexing is tested in TestCrossChainTokensIndexingFromMainChain +} diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index ca18d60d..075516a6 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -34,7 +34,7 @@ type ArgsIndexerFactory struct { ImportDB bool Sovereign bool ESDTPrefix string - MainChainElastic factory.MainChainElastic + MainChainElastic factory.ElasticConfig Denomination int BulkRequestMaxSize int Url string From 9659e73226114e15095c4e9abe793f7c9cacf9de Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 27 Jan 2025 18:17:42 +0200 Subject: [PATCH 05/17] fixes after self review --- integrationtests/testdata/incomingSCR/incoming-scr.json | 2 +- process/elasticproc/tokens/indexTokensHandler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json index 29253e9a..4b53272c 100644 --- a/integrationtests/testdata/incomingSCR/incoming-scr.json +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -40,4 +40,4 @@ 0 ], "operation": "MultiESDTNFTTransfer" -} \ No newline at end of file +} diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go index d8d78e7c..2f45c134 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -12,7 +12,7 @@ func NewIndexTokensHandler() *indexTokensHandler { return &indexTokensHandler{} } -// IndexCrossChainTokens returns no error +// IndexCrossChainTokens should do nothing and return no error func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { return nil } From 372b46ae2461439a017d9ec2b1629992a2658546 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 11:21:09 +0200 Subject: [PATCH 06/17] integration test fixes --- factory/runType/runTypeComponents_test.go | 6 ++- .../sovereignRunTypeComponentsFactory_test.go | 7 ++- integrationtests/consts.go | 3 +- integrationtests/incomingSCR_test.go | 53 +++++++++++-------- .../testdata/incomingSCR/incoming-scr.json | 2 +- .../tokens/sovereignIndexTokensHandler.go | 14 +++-- 6 files changed, 51 insertions(+), 34 deletions(-) diff --git a/factory/runType/runTypeComponents_test.go b/factory/runType/runTypeComponents_test.go index 07580802..9f929ea2 100644 --- a/factory/runType/runTypeComponents_test.go +++ b/factory/runType/runTypeComponents_test.go @@ -21,8 +21,9 @@ func TestRunTypeComponentsFactory_Create(t *testing.T) { rtcf := NewRunTypeComponentsFactory() require.NotNil(t, rtcf) - rtc := rtcf.Create() + rtc, err := rtcf.Create() require.NotNil(t, rtc) + require.NoError(t, err) } func TestRunTypeComponentsFactory_Close(t *testing.T) { @@ -31,8 +32,9 @@ func TestRunTypeComponentsFactory_Close(t *testing.T) { rtcf := NewRunTypeComponentsFactory() require.NotNil(t, rtcf) - rtc := rtcf.Create() + rtc, err := rtcf.Create() require.NotNil(t, rtc) + require.NoError(t, err) require.NoError(t, rtc.Close()) } diff --git a/factory/runType/sovereignRunTypeComponentsFactory_test.go b/factory/runType/sovereignRunTypeComponentsFactory_test.go index 71d909ba..4bb3581b 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory_test.go +++ b/factory/runType/sovereignRunTypeComponentsFactory_test.go @@ -4,16 +4,19 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" ) func TestSovereignRunTypeComponentsFactory_CreateAndClose(t *testing.T) { t.Parallel() - srtcf := NewSovereignRunTypeComponentsFactory() + srtcf := NewSovereignRunTypeComponentsFactory(factory.ElasticConfig{}, "sov") require.False(t, srtcf.IsInterfaceNil()) - srtc := srtcf.Create() + srtc, err := srtcf.Create() require.NotNil(t, srtc) + require.NoError(t, err) require.NoError(t, srtc.Close()) } diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 9c68a127..f06babca 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -4,7 +4,8 @@ const ( //nolint testNumOfShards = 3 //nolint - esURL = "http://localhost:9200" + esURL = "http://localhost:9200" + //nolint es2URL = "http://localhost:9201" //nolint addressPrefix = "erd" diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index 41971e9d..e35d08b9 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -27,14 +27,13 @@ type esToken struct { NumDecimals int64 } -type esEsdt struct { - Identifier string +type esNft struct { + Collection string Nonce uint64 - EsdtType string Data esdt.ESDigitalToken } -func createTokens() ([]esToken, []esEsdt) { +func createTokens() ([]esToken, []esNft) { tokens := []esToken{} token1 := esToken{ Identifier: "TKN18-1a2b3c", @@ -49,18 +48,19 @@ func createTokens() ([]esToken, []esEsdt) { } tokens = append(tokens, token2) - nfts := []esEsdt{} - nft := esEsdt{ - Identifier: "NFT-abc123", + nfts := []esNft{} + nft := esNft{ + Collection: "NFT-abc123", Nonce: 1, - EsdtType: core.NonFungibleESDTv2, Data: esdt.ESDigitalToken{ Type: uint32(core.NonFungibleV2), Value: big.NewInt(1), Properties: []byte("3032"), TokenMetaData: &esdt.MetaData{ - Nonce: 1, - Creator: []byte("creator"), + Nonce: 1, + Name: []byte("NFT"), + Creator: []byte("creator"), + Royalties: uint32(2500), }, }, } @@ -73,7 +73,8 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() mainChainEs := factory.ElasticConfig{ - Url: es2URL, + Enabled: true, + Url: es2URL, } tokens, nfts := createTokens() @@ -86,6 +87,7 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { require.Nil(t, err) allTokens := getAllTokensIDs(tokens, nfts) + allTokens = append(allTokens, getAllNftIDs(nfts)...) genericResponse := &GenericResponse{} err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) require.Nil(t, err) @@ -141,7 +143,7 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { } } -func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esEsdt) { +func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esNft) { esClient, err := createESClient(es.Url) require.Nil(t, err) @@ -170,7 +172,7 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT events = append(events, &transaction.Event{ Address: decodeAddress(address1), Identifier: []byte("issueNonFungible"), - Topics: [][]byte{[]byte(nft.Identifier), []byte("NFT"), []byte("NFT"), []byte(nft.EsdtType)}, + Topics: [][]byte{[]byte(nft.Collection), []byte("NFT"), []byte("NFT"), []byte(core.ESDTType(nft.Data.Type).String())}, }) } @@ -205,7 +207,7 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT events = append(events, &transaction.Event{ Address: decodeAddress(address1), Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte(nft.Identifier), big.NewInt(int64(nft.Nonce)).Bytes(), nft.Data.Value.Bytes(), []byte(nftDataBytes)}, + Topics: [][]byte{[]byte(nft.Collection), big.NewInt(0).SetUint64(nft.Nonce).Bytes(), nft.Data.Value.Bytes(), []byte(nftDataBytes)}, }) } @@ -230,10 +232,7 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - allNfts := make([]string, 0) - for _, nft := range nfts { - allNfts = append(allNfts, nft.Identifier+"-"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())) - } + allNfts := getAllNftIDs(nfts) err = esClient.DoMultiGet(context.Background(), allNfts, indexerData.TokensIndex, true, genericResponse) require.Nil(t, err) for _, token := range genericResponse.Docs { @@ -241,18 +240,26 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT } } -func getAllTokensIDs(tokens []esToken, nfts []esEsdt) []string { +func getAllTokensIDs(tokens []esToken, nfts []esNft) []string { allTokens := make([]string, 0) for _, token := range tokens { allTokens = append(allTokens, token.Identifier) } for _, nft := range nfts { - allTokens = append(allTokens, nft.Identifier) + allTokens = append(allTokens, nft.Collection) } return allTokens } -func createMultiEsdtTransferData(tokens []esToken, nfts []esEsdt) []byte { +func getAllNftIDs(nfts []esNft) []string { + allNfts := make([]string, 0) + for _, nft := range nfts { + allNfts = append(allNfts, nft.Collection+"-"+hex.EncodeToString(big.NewInt(0).SetUint64(nft.Nonce).Bytes())) + } + return allNfts +} + +func createMultiEsdtTransferData(tokens []esToken, nfts []esNft) []byte { data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + "@" + hex.EncodeToString(big.NewInt(int64(len(tokens)+len(nfts))).Bytes())) for _, token := range tokens { @@ -264,8 +271,8 @@ func createMultiEsdtTransferData(tokens []esToken, nfts []esEsdt) []byte { for _, nft := range nfts { nftDataBytes, _ := json.Marshal(nft.Data) data = append(data, []byte( - "@"+hex.EncodeToString([]byte(nft.Identifier))+ - "@"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())+ + "@"+hex.EncodeToString([]byte(nft.Collection))+ + "@"+hex.EncodeToString(big.NewInt(0).SetUint64(nft.Nonce).Bytes())+ "@"+hex.EncodeToString(nftDataBytes))...) } diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json index 4b53272c..5f4c142e 100644 --- a/integrationtests/testdata/incomingSCR/incoming-scr.json +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -9,7 +9,7 @@ "receiver": "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "senderShard": 4294967293, "receiverShard": 0, - "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDNANTQ0YjRlMzEzODJkMzE2MTMyNjIzMzYzQEA3YkA1NDRiNGUzMTMyMmQzMTYzMzI2MjMzNjFAQDAxNGRANGU0NjU0MmQ2MTYyNjMzMTMyMzNAMDFAN2IyMjU0Nzk3MDY1MjIzYTMyMmMyMjU2NjE2Yzc1NjUyMjNhMzEyYzIyNTA3MjZmNzA2NTcyNzQ2OTY1NzMyMjNhMjI0ZDdhNDE3YTRkNjczZDNkMjIyYzIyNGQ2NTc0NjE0NDYxNzQ2MTIyM2E3YjIyNGU2ZjZlNjM2NTIyM2EzMTJjMjI0ZTYxNmQ2NTIyM2E2ZTc1NmM2YzJjMjI0MzcyNjU2MTc0NmY3MjIyM2EyMjU5MzM0YTZjNTk1ODUyNzY2MzY3M2QzZDIyMmMyMjUyNmY3OTYxNmM3NDY5NjU3MzIyM2EzMDJjMjI0ODYxNzM2ODIyM2E2ZTc1NmM2YzJjMjI1NTUyNDk3MzIyM2E2ZTc1NmM2YzJjMjI0MTc0NzQ3MjY5NjI3NTc0NjU3MzIyM2E2ZTc1NmM2YzdkMmMyMjUyNjU3MzY1NzI3NjY1NjQyMjNhNmU3NTZjNmM3ZA==", + "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDNANTQ0YjRlMzEzODJkMzE2MTMyNjIzMzYzQEA3YkA1NDRiNGUzMTMyMmQzMTYzMzI2MjMzNjFAQDAxNGRANGU0NjU0MmQ2MTYyNjMzMTMyMzNAMDFAN2IyMjU0Nzk3MDY1MjIzYTMyMmMyMjU2NjE2Yzc1NjUyMjNhMzEyYzIyNTA3MjZmNzA2NTcyNzQ2OTY1NzMyMjNhMjI0ZDdhNDE3YTRkNjczZDNkMjIyYzIyNGQ2NTc0NjE0NDYxNzQ2MTIyM2E3YjIyNGU2ZjZlNjM2NTIyM2EzMTJjMjI0ZTYxNmQ2NTIyM2EyMjU0NmI1YTU1MjIyYzIyNDM3MjY1NjE3NDZmNzIyMjNhMjI1OTMzNGE2YzU5NTg1Mjc2NjM2NzNkM2QyMjJjMjI1MjZmNzk2MTZjNzQ2OTY1NzMyMjNhMzIzNTMwMzAyYzIyNDg2MTczNjgyMjNhNmU3NTZjNmMyYzIyNTU1MjQ5NzMyMjNhNmU3NTZjNmMyYzIyNDE3NDc0NzI2OTYyNzU3NDY1NzMyMjNhNmU3NTZjNmM3ZDJjMjI1MjY1NzM2NTcyNzY2NTY0MjIzYTZlNzU2YzZjN2Q=", "prevTxHash": "", "originalTxHash": "", "callType": "0", diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index af9ebea0..0f145fc6 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -111,7 +111,7 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp newTokens := make([]string, 0) for _, token := range responseTokens.Docs { - if token.Found == false { + if !token.Found { newTokens = append(newTokens, token.ID) } } @@ -132,9 +132,9 @@ func getTokenCollection(hasPrefix bool, tokenIdentifier string) (bool, string) { func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { for _, responseToken := range responseTokensInfo { - token := formatToken(responseToken) + token, identifier := formatToken(responseToken) - meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, indexerdata.TokensIndex, converters.JsonEscape(token.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, indexerdata.TokensIndex, converters.JsonEscape(identifier), "\n")) serializedTokenData, err := json.Marshal(token) if err != nil { return err @@ -149,11 +149,15 @@ func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data return nil } -func formatToken(token data.ResponseTokenInfoDB) data.TokenInfo { +func formatToken(token data.ResponseTokenInfoDB) (data.TokenInfo, string) { token.Source.OwnersHistory = nil token.Source.Properties = nil - return token.Source + identifier := token.Source.Identifier // for NFTs + if identifier == "" { + identifier = token.Source.Token // for tokens/collections + } + return token.Source, identifier } // IsInterfaceNil returns true if there is no value under the interface From a110c90be057269d86e54b007afe4bde36a44930 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 11:59:44 +0200 Subject: [PATCH 07/17] unit test fixes --- process/elasticproc/check.go | 5 +++ process/elasticproc/elasticProcessor_test.go | 36 ++++++++++--------- .../factory/elasticProcessorFactory_test.go | 2 ++ process/elasticproc/indexTokenHandlerMock.go | 23 ++++++++++++ process/elasticproc/interface.go | 2 +- 5 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 process/elasticproc/indexTokenHandlerMock.go diff --git a/process/elasticproc/check.go b/process/elasticproc/check.go index 9284376e..b3953d15 100644 --- a/process/elasticproc/check.go +++ b/process/elasticproc/check.go @@ -2,7 +2,9 @@ package elasticproc import ( "github.com/multiversx/mx-chain-core-go/core/check" + elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) func checkArguments(arguments *ArgElasticProcessor) error { @@ -39,6 +41,9 @@ func checkArguments(arguments *ArgElasticProcessor) error { if check.IfNilReflect(arguments.OperationsProc) { return elasticIndexer.ErrNilOperationsHandler } + if check.IfNilReflect(arguments.IndexTokensHandler) { + return transactions.ErrNilIndexTokensHandler + } return nil } diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index b01180ac..424970cb 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -31,15 +31,16 @@ import ( func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, arguments *ArgElasticProcessor) *elasticProcessor { return &elasticProcessor{ - elasticClient: elasticsearchWriter, - enabledIndexes: arguments.EnabledIndexes, - blockProc: arguments.BlockProc, - transactionsProc: arguments.TransactionsProc, - miniblocksProc: arguments.MiniblocksProc, - accountsProc: arguments.AccountsProc, - validatorsProc: arguments.ValidatorsProc, - statisticsProc: arguments.StatisticsProc, - logsAndEventsProc: arguments.LogsAndEventsProc, + elasticClient: elasticsearchWriter, + enabledIndexes: arguments.EnabledIndexes, + blockProc: arguments.BlockProc, + transactionsProc: arguments.TransactionsProc, + miniblocksProc: arguments.MiniblocksProc, + accountsProc: arguments.AccountsProc, + validatorsProc: arguments.ValidatorsProc, + statisticsProc: arguments.StatisticsProc, + logsAndEventsProc: arguments.LogsAndEventsProc, + indexTokensHandler: arguments.IndexTokensHandler, } } @@ -80,14 +81,15 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { EnabledIndexes: map[string]struct{}{ dataindexer.BlockIndex: {}, dataindexer.TransactionsIndex: {}, dataindexer.MiniblocksIndex: {}, dataindexer.ValidatorsIndex: {}, dataindexer.RoundsIndex: {}, dataindexer.AccountsIndex: {}, dataindexer.RatingIndex: {}, dataindexer.AccountsHistoryIndex: {}, }, - ValidatorsProc: vp, - StatisticsProc: statistics.NewStatisticsProcessor(), - TransactionsProc: &mock.DBTransactionProcessorStub{}, - MiniblocksProc: mp, - AccountsProc: acp, - BlockProc: bp, - LogsAndEventsProc: lp, - OperationsProc: op, + ValidatorsProc: vp, + StatisticsProc: statistics.NewStatisticsProcessor(), + TransactionsProc: &mock.DBTransactionProcessorStub{}, + MiniblocksProc: mp, + AccountsProc: acp, + BlockProc: bp, + LogsAndEventsProc: lp, + OperationsProc: op, + IndexTokensHandler: &IndexTokenHandlerMock{}, } } diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 2e4dd081..f7271737 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/multiversx/mx-chain-es-indexer-go/mock" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) func TestCreateElasticProcessor(t *testing.T) { @@ -21,6 +22,7 @@ func TestCreateElasticProcessor(t *testing.T) { UseKibana: false, TxHashExtractor: &mock.TxHashExtractorMock{}, RewardTxData: &mock.RewardTxDataMock{}, + IndexTokensHandler: &elasticproc.IndexTokenHandlerMock{}, } ep, err := CreateElasticProcessor(args) diff --git a/process/elasticproc/indexTokenHandlerMock.go b/process/elasticproc/indexTokenHandlerMock.go new file mode 100644 index 00000000..e60f36fb --- /dev/null +++ b/process/elasticproc/indexTokenHandlerMock.go @@ -0,0 +1,23 @@ +package elasticproc + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/data" +) + +// IndexTokenHandlerMock - +type IndexTokenHandlerMock struct { + IndexCrossChainTokensCalled func(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error +} + +// IndexCrossChainTokens - +func (ithh *IndexTokenHandlerMock) IndexCrossChainTokens(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + if ithh.IndexCrossChainTokensCalled != nil { + return ithh.IndexCrossChainTokensCalled(elasticClient, scrs, buffSlice) + } + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (ithh *IndexTokenHandlerMock) IsInterfaceNil() bool { + return ithh == nil +} diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index fd5c0af0..129d458d 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -124,6 +124,6 @@ type OperationsHandler interface { // IndexTokensHandler defines what index tokens handler should be able to do type IndexTokensHandler interface { - IndexCrossChainTokens(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error + IndexCrossChainTokens(handler DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error IsInterfaceNil() bool } From a07073e68cf286705e1ee4021c7dc1dd49c2431c Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 12:05:27 +0200 Subject: [PATCH 08/17] integration test fixes --- integrationtests/valuesIndex_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/integrationtests/valuesIndex_test.go b/integrationtests/valuesIndex_test.go index f76835f7..0fbd7ae3 100644 --- a/integrationtests/valuesIndex_test.go +++ b/integrationtests/valuesIndex_test.go @@ -7,10 +7,12 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/mock" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - "github.com/stretchr/testify/require" ) func TestCheckVersionIsIndexer(t *testing.T) { @@ -27,6 +29,9 @@ func TestCheckVersionIsIndexer(t *testing.T) { Denomination: 18, Version: version, EnabledIndexes: []string{indexerData.ValuesIndex}, + TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, + IndexTokensHandler: &elasticproc.IndexTokenHandlerMock{}, } _, err = factory.CreateElasticProcessor(args) From 646fbbfbc7a85fcb64ad5bcb27cf4299b9680752 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 13:22:04 +0200 Subject: [PATCH 09/17] script.sh update --- scripts/script.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/script.sh b/scripts/script.sh index 11a1bdb4..ff224ef4 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,4 +1,5 @@ IMAGE_NAME=elastic-container +IMAGE_NAME_2=elastic-container-2 DEFAULT_ES_VERSION=7.16.2 PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container @@ -16,10 +17,15 @@ start() { docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} docker rm ${IMAGE_NAME} 2> /dev/null + docker rm ${IMAGE_NAME_2} 2> /dev/null docker run -d --name "${IMAGE_NAME}" -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + docker run -d --name "${IMAGE_NAME_2}" -p 9201:9200 -p 9301:9300 \ + -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + # Wait elastic cluster to start echo "Waiting Elasticsearch cluster to start..." sleep 30s @@ -27,6 +33,7 @@ start() { stop() { docker stop "${IMAGE_NAME}" + docker stop "${IMAGE_NAME_2}" } delete() { From 86bc0754b1e96773c85006fae2e09d1df88f1851 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 13:26:25 +0200 Subject: [PATCH 10/17] script.sh update --- scripts/script.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/script.sh b/scripts/script.sh index ff224ef4..6328f90d 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -48,6 +48,7 @@ delete() { IMAGE_OPEN_SEARCH=open-container +IMAGE_OPEN_SEARCH_2=open-container-2 DEFAULT_OPEN_SEARCH_VERSION=1.2.4 start_open_search() { @@ -59,9 +60,13 @@ start_open_search() { docker pull opensearchproject/opensearch:${OPEN_VERSION} docker rm ${IMAGE_OPEN_SEARCH} 2> /dev/null + docker rm ${IMAGE_OPEN_SEARCH_2} 2> /dev/null docker run -d --name "${IMAGE_OPEN_SEARCH}" -p 9200:9200 -p 9600:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} + docker run -d --name "${IMAGE_OPEN_SEARCH_2}" -p 9201:9200 -p 9601:9600 \ + -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + opensearchproject/opensearch:${OPEN_VERSION} } From 87cc67fbce7dc3640c20d8ad579f722d159c67f9 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 5 Feb 2025 12:03:52 +0200 Subject: [PATCH 11/17] fixes after review --- client/disabled/elasticClient.go | 73 +++++++++++++++++++ client/disabled/elasticClient_test.go | 30 ++++++++ data/tokens.go | 2 +- docker-compose.yml | 2 +- factory/runType/runTypeComponentsFactory.go | 2 +- factory/runType/runTypeComponentsHandler.go | 2 +- .../sovereignRunTypeComponentsFactory.go | 37 +++++++++- integrationtests/consts.go | 2 +- integrationtests/incomingSCR_test.go | 24 +++--- integrationtests/utils.go | 6 +- .../elasticproc/tokens/indexTokensHandler.go | 4 +- .../tokens/indexTokensHandler_test.go | 4 +- .../tokens/sovereignIndexTokensHandler.go | 73 +++++++------------ .../sovereignIndexTokensHandler_test.go | 38 +++------- scripts/script.sh | 14 ++-- 15 files changed, 205 insertions(+), 108 deletions(-) create mode 100644 client/disabled/elasticClient.go create mode 100644 client/disabled/elasticClient_test.go diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go new file mode 100644 index 00000000..eba23d28 --- /dev/null +++ b/client/disabled/elasticClient.go @@ -0,0 +1,73 @@ +package disabled + +import ( + "bytes" + "context" +) + +type elasticClient struct{} + +// NewDisabledElasticClient - +func NewDisabledElasticClient() *elasticClient { + return &elasticClient{} +} + +// DoBulkRequest - +func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { + return nil +} + +// DoQueryRemove - +func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// DoMultiGet - +func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { + return nil +} + +// DoScrollRequest - +func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { + return 0, nil +} + +// UpdateByQuery - +func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// PutMappings - +func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreateIndex - +func (ec *elasticClient) CheckAndCreateIndex(_ string) error { + return nil +} + +// CheckAndCreateAlias - +func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error { + return nil +} + +// CheckAndCreateTemplate - +func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreatePolicy - +func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { + return nil +} + +// IsInterfaceNil - returns true if there is no value under the interface +func (ec *elasticClient) IsInterfaceNil() bool { + return ec == nil +} diff --git a/client/disabled/elasticClient_test.go b/client/disabled/elasticClient_test.go new file mode 100644 index 00000000..cae05ffd --- /dev/null +++ b/client/disabled/elasticClient_test.go @@ -0,0 +1,30 @@ +package disabled + +import ( + "bytes" + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/stretchr/testify/require" +) + +func TestDisabledElasticClient_MethodsShouldNotPanic(t *testing.T) { + t.Parallel() + + ec := NewDisabledElasticClient() + require.False(t, check.IfNil(ec)) + + require.NotPanics(t, func() { + _ = ec.DoBulkRequest(nil, new(bytes.Buffer), "") + _ = ec.DoQueryRemove(nil, "", new(bytes.Buffer)) + _ = ec.DoMultiGet(nil, make([]string, 0), "", true, nil) + _ = ec.DoScrollRequest(nil, "", []byte(""), true, nil) + _, _ = ec.DoCountRequest(nil, "", []byte("")) + _ = ec.UpdateByQuery(nil, "", new(bytes.Buffer)) + _ = ec.PutMappings("", new(bytes.Buffer)) + _ = ec.CheckAndCreateIndex("") + _ = ec.CheckAndCreateAlias("", "") + _ = ec.CheckAndCreateTemplate("", new(bytes.Buffer)) + _ = ec.CheckAndCreatePolicy("", new(bytes.Buffer)) + }) +} diff --git a/data/tokens.go b/data/tokens.go index d70b6df9..01f39dce 100644 --- a/data/tokens.go +++ b/data/tokens.go @@ -45,7 +45,7 @@ type SourceToken struct { CurrentOwner string `json:"currentOwner"` } -// ResponseTokenInfo is the structure for the token info response +// ResponseTokenInfo is the structure for the tokens info response type ResponseTokenInfo struct { Docs []ResponseTokenInfoDB `json:"docs"` } diff --git a/docker-compose.yml b/docker-compose.yml index a5a27b1f..ba288828 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "9200:9200" - "9300:9300" - elasticsearch2: + main-elasticsearch: container_name: es-container2 image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 environment: diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go index af7d42a3..8ac863d6 100644 --- a/factory/runType/runTypeComponentsFactory.go +++ b/factory/runType/runTypeComponentsFactory.go @@ -17,7 +17,7 @@ func (rtcf *runTypeComponentsFactory) Create() (*runTypeComponents, error) { return &runTypeComponents{ txHashExtractor: transactions.NewTxHashExtractor(), rewardTxData: transactions.NewRewardTxData(), - indexTokensHandler: tokens.NewIndexTokensHandler(), + indexTokensHandler: tokens.NewDisabledIndexTokensHandler(), }, nil } diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 1d471a5c..b9697008 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -109,7 +109,7 @@ func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardT return mrtc.runTypeComponents.rewardTxData } -// IndexTokensHandlerCreator return index tokens handler +// IndexTokensHandlerCreator returns the index tokens handler func (mrtc *managedRunTypeComponents) IndexTokensHandlerCreator() elasticproc.IndexTokensHandler { mrtc.mutRunTypeCoreComponents.Lock() defer mrtc.mutRunTypeCoreComponents.Unlock() diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 2adb96c7..54590352 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -1,6 +1,16 @@ package runType import ( + "math" + "net/http" + "time" + + "github.com/elastic/go-elasticsearch/v7" + + "github.com/multiversx/mx-chain-es-indexer-go/client" + "github.com/multiversx/mx-chain-es-indexer-go/client/disabled" + "github.com/multiversx/mx-chain-es-indexer-go/client/logging" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" @@ -21,7 +31,12 @@ func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.ElasticConfig // Create will create the run type components func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, error) { - sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic, srtcf.esdtPrefix) + mainChainElasticClient, err := createMainChainElasticClient(srtcf.mainChainElastic) + if err != nil { + return nil, err + } + + sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic.Enabled, mainChainElasticClient, srtcf.esdtPrefix) if err != nil { return nil, err } @@ -33,6 +48,26 @@ func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, er }, nil } +func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elasticproc.DatabaseClientHandler, error) { + if mainChainElastic.Enabled { + argsEsClient := elasticsearch.Config{ + Addresses: []string{mainChainElastic.Url}, + Username: mainChainElastic.UserName, + Password: mainChainElastic.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: retryBackOff, + } + return client.NewElasticClient(argsEsClient) + } else { + return disabled.NewDisabledElasticClient(), nil + } +} + +func retryBackOff(attempt int) time.Duration { + return time.Duration(math.Exp2(float64(attempt))) * time.Second +} + // IsInterfaceNil returns true if there is no value under the interface func (srtcf *sovereignRunTypeComponentsFactory) IsInterfaceNil() bool { return srtcf == nil diff --git a/integrationtests/consts.go b/integrationtests/consts.go index f06babca..34d0ca79 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -6,7 +6,7 @@ const ( //nolint esURL = "http://localhost:9200" //nolint - es2URL = "http://localhost:9201" + esMainURL = "http://localhost:9201" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index e35d08b9..612a2b60 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/require" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) type esToken struct { @@ -72,18 +72,16 @@ func createTokens() ([]esToken, []esNft) { func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() - mainChainEs := factory.ElasticConfig{ - Enabled: true, - Url: es2URL, - } + mainEsClient, err := createESClient(esMainURL) + require.Nil(t, err) tokens, nfts := createTokens() - createTokensInSourceEs(t, mainChainEs, tokens, nfts) + createTokensInSourceEs(t, mainEsClient, tokens, nfts) esClient, err := createESClient(esURL) require.Nil(t, err) - esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEs) + esProc, err := CreateSovereignElasticProcessor(esClient, mainEsClient) require.Nil(t, err) allTokens := getAllTokensIDs(tokens, nfts) @@ -143,10 +141,7 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { } } -func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esNft) { - esClient, err := createESClient(es.Url) - require.Nil(t, err) - +func createTokensInSourceEs(t *testing.T, esClient elasticproc.DatabaseClientHandler, tokens []esToken, nfts []esNft) { esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) @@ -254,7 +249,12 @@ func getAllTokensIDs(tokens []esToken, nfts []esNft) []string { func getAllNftIDs(nfts []esNft) []string { allNfts := make([]string, 0) for _, nft := range nfts { - allNfts = append(allNfts, nft.Collection+"-"+hex.EncodeToString(big.NewInt(0).SetUint64(nft.Nonce).Bytes())) + nonceBytes := big.NewInt(0).SetUint64(nft.Nonce).Bytes() + nonceHex := hex.EncodeToString(nonceBytes) + nftIdentifier := nft.Collection + "-" + nonceHex + + allNfts = append(allNfts, nftIdentifier) + } return allNfts } diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 0577ca81..93ab9924 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -67,7 +67,7 @@ func CreateElasticProcessor( Denomination: 18, TxHashExtractor: transactions.NewTxHashExtractor(), RewardTxData: transactions.NewRewardTxData(), - IndexTokensHandler: tokens.NewIndexTokensHandler(), + IndexTokensHandler: tokens.NewDisabledIndexTokensHandler(), } return factory.CreateElasticProcessor(args) @@ -76,9 +76,9 @@ func CreateElasticProcessor( // CreateSovereignElasticProcessor - func CreateSovereignElasticProcessor( esClient elasticproc.DatabaseClientHandler, - mainChainEs factory.ElasticConfig, + mainEsClient elasticproc.DatabaseClientHandler, ) (dataindexer.ElasticProcessor, error) { - sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainChainEs, sovEsdtPrefix) + sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(true, mainEsClient, sovEsdtPrefix) args := factory.ArgElasticProcessorFactory{ Marshalizer: &mock.MarshalizerMock{}, diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go index 2f45c134..99a4b8c4 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -7,8 +7,8 @@ import ( type indexTokensHandler struct{} -// NewIndexTokensHandler creates a new index tokens handler -func NewIndexTokensHandler() *indexTokensHandler { +// NewDisabledIndexTokensHandler creates a new disabled index tokens handler +func NewDisabledIndexTokensHandler() *indexTokensHandler { return &indexTokensHandler{} } diff --git a/process/elasticproc/tokens/indexTokensHandler_test.go b/process/elasticproc/tokens/indexTokensHandler_test.go index 260c39af..6b490a81 100644 --- a/process/elasticproc/tokens/indexTokensHandler_test.go +++ b/process/elasticproc/tokens/indexTokensHandler_test.go @@ -9,14 +9,14 @@ import ( func TestNewIndexTokensHandler(t *testing.T) { t.Parallel() - ith := NewIndexTokensHandler() + ith := NewDisabledIndexTokensHandler() require.False(t, ith.IsInterfaceNil()) } func TestIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { t.Parallel() - ith := NewIndexTokensHandler() + ith := NewDisabledIndexTokensHandler() err := ith.IndexCrossChainTokens(nil, nil, nil) require.NoError(t, err) } diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index 0f145fc6..0f9fb90b 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -4,22 +4,15 @@ import ( "context" "encoding/json" "fmt" - "math" - "net/http" "strings" - "time" - "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/esdt" - "github.com/multiversx/mx-chain-es-indexer-go/client" - "github.com/multiversx/mx-chain-es-indexer-go/client/logging" "github.com/multiversx/mx-chain-es-indexer-go/data" indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" ) type sovereignIndexTokensHandler struct { @@ -29,53 +22,32 @@ type sovereignIndexTokensHandler struct { } // NewSovereignIndexTokensHandler creates a new sovereign index tokens handler -func NewSovereignIndexTokensHandler(mainChainElastic factory.ElasticConfig, esdtPrefix string) (*sovereignIndexTokensHandler, error) { - var mainChainElasticClient elasticproc.DatabaseClientHandler - if mainChainElastic.Enabled { - var err error - argsEsClient := elasticsearch.Config{ - Addresses: []string{mainChainElastic.Url}, - Username: mainChainElastic.UserName, - Password: mainChainElastic.Password, - Logger: &logging.CustomLogger{}, - RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, - } - mainChainElasticClient, err = client.NewElasticClient(argsEsClient) - if err != nil { - return nil, err - } - } - +func NewSovereignIndexTokensHandler(indexingEnabled bool, mainChainElasticClient elasticproc.DatabaseClientHandler, esdtPrefix string) (*sovereignIndexTokensHandler, error) { return &sovereignIndexTokensHandler{ - indexingEnabled: mainChainElastic.Enabled, + indexingEnabled: indexingEnabled, mainChainElasticClient: mainChainElasticClient, esdtPrefix: esdtPrefix, }, nil } -func retryBackOff(attempt int) time.Duration { - return time.Duration(math.Exp2(float64(attempt))) * time.Second -} - // IndexCrossChainTokens will index the new tokens properties func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { if !sit.indexingEnabled { return nil } - notFoundTokens, err := sit.getTokensFromScrs(elasticClient, scrs) + newTokens, err := sit.getNewTokensFromSCRs(elasticClient, scrs) if err != nil { return err } - if len(notFoundTokens) == 0 { // no new tokens + if len(newTokens) == 0 { // no new tokens return nil } // get tokens from main chain elastic db mainChainTokens := &data.ResponseTokenInfo{} - err = sit.mainChainElasticClient.DoMultiGet(context.Background(), notFoundTokens, indexerdata.TokensIndex, true, mainChainTokens) + err = sit.mainChainElasticClient.DoMultiGet(context.Background(), newTokens, indexerdata.TokensIndex, true, mainChainTokens) if err != nil { return err } @@ -83,19 +55,11 @@ func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elas return sit.indexNewTokens(mainChainTokens.Docs, buffSlice) } -func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { +func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { receivedTokensIDs := make([]string, 0) for _, scr := range scrs { if scr.SenderShard == core.MainChainShardId { - for _, token := range scr.Tokens { - tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) - if !hasPrefix || tokenPrefix != sit.esdtPrefix { - receivedTokensIDs = append(receivedTokensIDs, token) - } - if isEsdt, tokenCollection := getTokenCollection(hasPrefix, token); isEsdt { - receivedTokensIDs = append(receivedTokensIDs, tokenCollection) - } - } + receivedTokensIDs = append(receivedTokensIDs, sit.extractSovereignTokens(scr.Tokens)...) } } @@ -119,15 +83,30 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp return newTokens, nil } -func getTokenCollection(hasPrefix bool, tokenIdentifier string) (bool, string) { +func (sit *sovereignIndexTokensHandler) extractSovereignTokens(tokens []string) []string { + receivedTokensIDs := make([]string, 0) + for _, token := range tokens { + tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) + if !hasPrefix || tokenPrefix != sit.esdtPrefix { + receivedTokensIDs = append(receivedTokensIDs, token) + } + if tokenCollection := getTokenCollection(hasPrefix, token); tokenCollection != "" { + receivedTokensIDs = append(receivedTokensIDs, tokenCollection) + } + } + + return receivedTokensIDs +} + +func getTokenCollection(hasPrefix bool, tokenIdentifier string) string { tokenSplit := strings.Split(tokenIdentifier, "-") if !hasPrefix && len(tokenSplit) == 3 { - return true, tokenSplit[0] + "-" + tokenSplit[1] + return tokenSplit[0] + "-" + tokenSplit[1] } if hasPrefix && len(tokenSplit) == 4 { - return true, tokenSplit[1] + "-" + tokenSplit[2] + return tokenSplit[1] + "-" + tokenSplit[2] } - return false, "" + return "" } func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go index 3aebe667..2cb86b88 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go @@ -1,59 +1,39 @@ package tokens import ( + "fmt" "testing" "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/client/disabled" "github.com/multiversx/mx-chain-es-indexer-go/data" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/mock" ) const ( prefix = "sov" ) -func createElasticConfig() factory.ElasticConfig { - return factory.ElasticConfig{ - Enabled: true, - Url: "http://localhost:9200", - UserName: "", - Password: "", - } -} - func TestSovereignNewIndexTokensHandler(t *testing.T) { t.Parallel() - t.Run("no url, should error", func(t *testing.T) { - esConfig := createElasticConfig() - esConfig.Url = "http://bad url" - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) - require.ErrorContains(t, err, "cannot parse url") - require.Nil(t, sith) - }) - t.Run("not enabled should not create main chain elastic client", func(t *testing.T) { - esConfig := createElasticConfig() - esConfig.Enabled = false - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + t.Run("valid disabled config, should work", func(t *testing.T) { + sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) - require.NotNil(t, sith) - require.Nil(t, sith.mainChainElasticClient) + require.Equal(t, "*disabled.elasticClient", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) t.Run("valid config, should work", func(t *testing.T) { - esConfig := createElasticConfig() - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + sith, err := NewSovereignIndexTokensHandler(true, &mock.DatabaseWriterStub{}, prefix) require.NoError(t, err) - require.NotNil(t, sith) + require.Equal(t, "*mock.DatabaseWriterStub", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) } func TestSovereignIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { t.Parallel() - esConfig := createElasticConfig() - esConfig.Enabled = false - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) require.NotNil(t, sith) diff --git a/scripts/script.sh b/scripts/script.sh index 6328f90d..2b58ebb3 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,5 +1,5 @@ IMAGE_NAME=elastic-container -IMAGE_NAME_2=elastic-container-2 +MAIN_IMAGE_NAME=main-elastic-container DEFAULT_ES_VERSION=7.16.2 PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container @@ -17,12 +17,12 @@ start() { docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} docker rm ${IMAGE_NAME} 2> /dev/null - docker rm ${IMAGE_NAME_2} 2> /dev/null + docker rm ${MAIN_IMAGE_NAME} 2> /dev/null docker run -d --name "${IMAGE_NAME}" -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} - docker run -d --name "${IMAGE_NAME_2}" -p 9201:9200 -p 9301:9300 \ + docker run -d --name "${MAIN_IMAGE_NAME}" -p 9201:9200 -p 9301:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} @@ -33,7 +33,7 @@ start() { stop() { docker stop "${IMAGE_NAME}" - docker stop "${IMAGE_NAME_2}" + docker stop "${MAIN_IMAGE_NAME}" } delete() { @@ -48,7 +48,7 @@ delete() { IMAGE_OPEN_SEARCH=open-container -IMAGE_OPEN_SEARCH_2=open-container-2 +MAIN_IMAGE_OPEN_SEARCH=main-open-container DEFAULT_OPEN_SEARCH_VERSION=1.2.4 start_open_search() { @@ -60,11 +60,11 @@ start_open_search() { docker pull opensearchproject/opensearch:${OPEN_VERSION} docker rm ${IMAGE_OPEN_SEARCH} 2> /dev/null - docker rm ${IMAGE_OPEN_SEARCH_2} 2> /dev/null + docker rm ${MAIN_IMAGE_OPEN_SEARCH} 2> /dev/null docker run -d --name "${IMAGE_OPEN_SEARCH}" -p 9200:9200 -p 9600:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} - docker run -d --name "${IMAGE_OPEN_SEARCH_2}" -p 9201:9200 -p 9601:9600 \ + docker run -d --name "${MAIN_IMAGE_OPEN_SEARCH}" -p 9201:9200 -p 9601:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} From 9d96aee965ad9bc7d052bc536cdd8205bc865916 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 5 Feb 2025 12:20:22 +0200 Subject: [PATCH 12/17] unit test fixes --- client/disabled/elasticClient_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client/disabled/elasticClient_test.go b/client/disabled/elasticClient_test.go index cae05ffd..c33e8716 100644 --- a/client/disabled/elasticClient_test.go +++ b/client/disabled/elasticClient_test.go @@ -2,6 +2,7 @@ package disabled import ( "bytes" + "context" "testing" "github.com/multiversx/mx-chain-core-go/core/check" @@ -15,12 +16,12 @@ func TestDisabledElasticClient_MethodsShouldNotPanic(t *testing.T) { require.False(t, check.IfNil(ec)) require.NotPanics(t, func() { - _ = ec.DoBulkRequest(nil, new(bytes.Buffer), "") - _ = ec.DoQueryRemove(nil, "", new(bytes.Buffer)) - _ = ec.DoMultiGet(nil, make([]string, 0), "", true, nil) - _ = ec.DoScrollRequest(nil, "", []byte(""), true, nil) - _, _ = ec.DoCountRequest(nil, "", []byte("")) - _ = ec.UpdateByQuery(nil, "", new(bytes.Buffer)) + _ = ec.DoBulkRequest(context.Background(), new(bytes.Buffer), "") + _ = ec.DoQueryRemove(context.Background(), "", new(bytes.Buffer)) + _ = ec.DoMultiGet(context.Background(), make([]string, 0), "", true, nil) + _ = ec.DoScrollRequest(context.Background(), "", []byte(""), true, nil) + _, _ = ec.DoCountRequest(context.Background(), "", []byte("")) + _ = ec.UpdateByQuery(context.Background(), "", new(bytes.Buffer)) _ = ec.PutMappings("", new(bytes.Buffer)) _ = ec.CheckAndCreateIndex("") _ = ec.CheckAndCreateAlias("", "") From d2c8827aa9c4a9abd29933ab89c765230b113727 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 6 Feb 2025 10:42:56 +0200 Subject: [PATCH 13/17] fixes after review --- client/disabled/elasticClient.go | 5 +++ client/mainChainElasticClient.go | 28 +++++++++++++++ client/mainChainElasticClient_test.go | 36 +++++++++++++++++++ cmd/elasticindexer/config/prefs.toml | 2 +- config/config.go | 2 +- docker-compose.yml | 2 +- .../sovereignRunTypeComponentsFactory.go | 6 ++-- integrationtests/consts.go | 2 +- integrationtests/incomingSCR_test.go | 6 ++-- integrationtests/utils.go | 12 +++++-- mock/databaseWriterStub.go | 5 +++ process/elasticproc/interface.go | 6 ++++ .../tokens/sovereignIndexTokensHandler.go | 12 +++---- .../sovereignIndexTokensHandler_test.go | 6 ++-- scripts/script.sh | 19 +++++----- 15 files changed, 117 insertions(+), 32 deletions(-) create mode 100644 client/mainChainElasticClient.go create mode 100644 client/mainChainElasticClient_test.go diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go index eba23d28..41ccc6c0 100644 --- a/client/disabled/elasticClient.go +++ b/client/disabled/elasticClient.go @@ -67,6 +67,11 @@ func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { return nil } +// IsEnabled - +func (ec *elasticClient) IsEnabled() bool { + return false +} + // IsInterfaceNil - returns true if there is no value under the interface func (ec *elasticClient) IsInterfaceNil() bool { return ec == nil diff --git a/client/mainChainElasticClient.go b/client/mainChainElasticClient.go new file mode 100644 index 00000000..e378dd9a --- /dev/null +++ b/client/mainChainElasticClient.go @@ -0,0 +1,28 @@ +package client + +import ( + "github.com/elastic/go-elasticsearch/v7" +) + +type mainChainElasticClient struct { + *elasticClient + indexingEnabled bool +} + +// NewMainChainElasticClient creates a new sovereign elastic client +func NewMainChainElasticClient(cfg elasticsearch.Config, indexingEnabled bool) (*mainChainElasticClient, error) { + esClient, err := NewElasticClient(cfg) + if err != nil { + return nil, err + } + + return &mainChainElasticClient{ + esClient, + indexingEnabled, + }, nil +} + +// IsEnabled returns true if main chain elastic client is enabled +func (mcec *mainChainElasticClient) IsEnabled() bool { + return mcec.indexingEnabled +} diff --git a/client/mainChainElasticClient_test.go b/client/mainChainElasticClient_test.go new file mode 100644 index 00000000..c72c5079 --- /dev/null +++ b/client/mainChainElasticClient_test.go @@ -0,0 +1,36 @@ +package client + +import ( + "fmt" + "testing" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/stretchr/testify/require" + + indexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" +) + +func TestNewMainChainElasticClient(t *testing.T) { + t.Run("no url, should error", func(t *testing.T) { + esClient, err := NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{}, + }, true) + require.Nil(t, esClient) + require.Equal(t, indexer.ErrNoElasticUrlProvided, err) + }) + t.Run("should work", func(t *testing.T) { + esClient, err := NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + }, true) + require.Nil(t, err) + require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", esClient)) + }) +} + +func TestMainChainElasticClient_IsEnabled(t *testing.T) { + esClient, err := NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + }, true) + require.Nil(t, err) + require.Equal(t, true, esClient.IsEnabled()) +} diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 7c320e89..42dc095f 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -24,7 +24,7 @@ password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB - [config.main-elastic-cluster] + [config.main-chain-elastic-cluster] enabled = true url = "http://localhost:9201" username = "" diff --git a/config/config.go b/config/config.go index b5d3b83a..60c56b4d 100644 --- a/config/config.go +++ b/config/config.go @@ -58,7 +58,7 @@ type ClusterConfig struct { URL string `toml:"url"` UserName string `toml:"username"` Password string `toml:"password"` - } `toml:"main-elastic-cluster"` + } `toml:"main-chain-elastic-cluster"` } `toml:"config"` } diff --git a/docker-compose.yml b/docker-compose.yml index ba288828..27e689c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "9200:9200" - "9300:9300" - main-elasticsearch: + main-chain-elasticsearch: container_name: es-container2 image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 environment: diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 54590352..62aa1e2d 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -36,7 +36,7 @@ func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, er return nil, err } - sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic.Enabled, mainChainElasticClient, srtcf.esdtPrefix) + sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(mainChainElasticClient, srtcf.esdtPrefix) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, er }, nil } -func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elasticproc.DatabaseClientHandler, error) { +func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elasticproc.MainChainDatabaseClientHandler, error) { if mainChainElastic.Enabled { argsEsClient := elasticsearch.Config{ Addresses: []string{mainChainElastic.Url}, @@ -58,7 +58,7 @@ func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elast RetryOnStatus: []int{http.StatusConflict}, RetryBackoff: retryBackOff, } - return client.NewElasticClient(argsEsClient) + return client.NewMainChainElasticClient(argsEsClient, mainChainElastic.Enabled) } else { return disabled.NewDisabledElasticClient(), nil } diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 34d0ca79..32b4bd1b 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -6,7 +6,7 @@ const ( //nolint esURL = "http://localhost:9200" //nolint - esMainURL = "http://localhost:9201" + esMainChainURL = "http://localhost:9201" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index 612a2b60..b6931a0d 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -72,16 +72,16 @@ func createTokens() ([]esToken, []esNft) { func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() - mainEsClient, err := createESClient(esMainURL) + mainChainEsClient, err := createMainChainESClient(esMainChainURL, true) require.Nil(t, err) tokens, nfts := createTokens() - createTokensInSourceEs(t, mainEsClient, tokens, nfts) + createTokensInSourceEs(t, mainChainEsClient, tokens, nfts) esClient, err := createESClient(esURL) require.Nil(t, err) - esProc, err := CreateSovereignElasticProcessor(esClient, mainEsClient) + esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEsClient) require.Nil(t, err) allTokens := getAllTokensIDs(tokens, nfts) diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 93ab9924..796c4ac1 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -43,6 +43,14 @@ func createESClient(url string) (elasticproc.DatabaseClientHandler, error) { }) } +// nolint +func createMainChainESClient(url string, enabled bool) (elasticproc.MainChainDatabaseClientHandler, error) { + return client.NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{url}, + Logger: &logging.CustomLogger{}, + }, enabled) +} + // nolint func decodeAddress(address string) []byte { decoded, err := pubKeyConverter.Decode(address) @@ -76,9 +84,9 @@ func CreateElasticProcessor( // CreateSovereignElasticProcessor - func CreateSovereignElasticProcessor( esClient elasticproc.DatabaseClientHandler, - mainEsClient elasticproc.DatabaseClientHandler, + mainEsClient elasticproc.MainChainDatabaseClientHandler, ) (dataindexer.ElasticProcessor, error) { - sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(true, mainEsClient, sovEsdtPrefix) + sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainEsClient, sovEsdtPrefix) args := factory.ArgElasticProcessorFactory{ Marshalizer: &mock.MarshalizerMock{}, diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 1df608a8..5a0153ce 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -86,6 +86,11 @@ func (dwm *DatabaseWriterStub) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) e return nil } +// IsEnabled - +func (dwm *DatabaseWriterStub) IsEnabled() bool { + return false +} + // IsInterfaceNil returns true if there is no value under the interface func (dwm *DatabaseWriterStub) IsInterfaceNil() bool { return dwm == nil diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 129d458d..b5108f9b 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -13,6 +13,12 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) +// MainChainDatabaseClientHandler defines the actions that sovereign database client handler should do +type MainChainDatabaseClientHandler interface { + DatabaseClientHandler + IsEnabled() bool +} + // DatabaseClientHandler defines the actions that a component that handles requests should do type DatabaseClientHandler interface { DoBulkRequest(ctx context.Context, buff *bytes.Buffer, index string) error diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index 0f9fb90b..d814f31c 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -16,15 +16,13 @@ import ( ) type sovereignIndexTokensHandler struct { - indexingEnabled bool - mainChainElasticClient elasticproc.DatabaseClientHandler + mainChainElasticClient elasticproc.MainChainDatabaseClientHandler esdtPrefix string } // NewSovereignIndexTokensHandler creates a new sovereign index tokens handler -func NewSovereignIndexTokensHandler(indexingEnabled bool, mainChainElasticClient elasticproc.DatabaseClientHandler, esdtPrefix string) (*sovereignIndexTokensHandler, error) { +func NewSovereignIndexTokensHandler(mainChainElasticClient elasticproc.MainChainDatabaseClientHandler, esdtPrefix string) (*sovereignIndexTokensHandler, error) { return &sovereignIndexTokensHandler{ - indexingEnabled: indexingEnabled, mainChainElasticClient: mainChainElasticClient, esdtPrefix: esdtPrefix, }, nil @@ -32,7 +30,7 @@ func NewSovereignIndexTokensHandler(indexingEnabled bool, mainChainElasticClient // IndexCrossChainTokens will index the new tokens properties func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { - if !sit.indexingEnabled { + if !sit.mainChainElasticClient.IsEnabled() { return nil } @@ -59,7 +57,7 @@ func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elast receivedTokensIDs := make([]string, 0) for _, scr := range scrs { if scr.SenderShard == core.MainChainShardId { - receivedTokensIDs = append(receivedTokensIDs, sit.extractSovereignTokens(scr.Tokens)...) + receivedTokensIDs = append(receivedTokensIDs, sit.extractNewSovereignTokens(scr.Tokens)...) } } @@ -83,7 +81,7 @@ func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elast return newTokens, nil } -func (sit *sovereignIndexTokensHandler) extractSovereignTokens(tokens []string) []string { +func (sit *sovereignIndexTokensHandler) extractNewSovereignTokens(tokens []string) []string { receivedTokensIDs := make([]string, 0) for _, token := range tokens { tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go index 2cb86b88..b881289b 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go @@ -19,12 +19,12 @@ func TestSovereignNewIndexTokensHandler(t *testing.T) { t.Parallel() t.Run("valid disabled config, should work", func(t *testing.T) { - sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) + sith, err := NewSovereignIndexTokensHandler(disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) require.Equal(t, "*disabled.elasticClient", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) t.Run("valid config, should work", func(t *testing.T) { - sith, err := NewSovereignIndexTokensHandler(true, &mock.DatabaseWriterStub{}, prefix) + sith, err := NewSovereignIndexTokensHandler(&mock.DatabaseWriterStub{}, prefix) require.NoError(t, err) require.Equal(t, "*mock.DatabaseWriterStub", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) @@ -33,7 +33,7 @@ func TestSovereignNewIndexTokensHandler(t *testing.T) { func TestSovereignIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { t.Parallel() - sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) + sith, err := NewSovereignIndexTokensHandler(disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) require.NotNil(t, sith) diff --git a/scripts/script.sh b/scripts/script.sh index 2b58ebb3..166780fc 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,5 +1,5 @@ IMAGE_NAME=elastic-container -MAIN_IMAGE_NAME=main-elastic-container +MAIN_CHAIN_IMAGE_NAME=main-chain-elastic-container DEFAULT_ES_VERSION=7.16.2 PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container @@ -17,14 +17,13 @@ start() { docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} docker rm ${IMAGE_NAME} 2> /dev/null - docker rm ${MAIN_IMAGE_NAME} 2> /dev/null + docker rm ${MAIN_CHAIN_IMAGE_NAME} 2> /dev/null docker run -d --name "${IMAGE_NAME}" -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} - - docker run -d --name "${MAIN_IMAGE_NAME}" -p 9201:9200 -p 9301:9300 \ - -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ - docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + docker run -d --name "${MAIN_CHAIN_IMAGE_NAME}" -p 9201:9200 -p 9301:9300 \ + -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} # Wait elastic cluster to start echo "Waiting Elasticsearch cluster to start..." @@ -33,7 +32,7 @@ start() { stop() { docker stop "${IMAGE_NAME}" - docker stop "${MAIN_IMAGE_NAME}" + docker stop "${MAIN_CHAIN_IMAGE_NAME}" } delete() { @@ -48,7 +47,7 @@ delete() { IMAGE_OPEN_SEARCH=open-container -MAIN_IMAGE_OPEN_SEARCH=main-open-container +MAIN_CHAIN_IMAGE_OPEN_SEARCH=main-chain-open-container DEFAULT_OPEN_SEARCH_VERSION=1.2.4 start_open_search() { @@ -60,11 +59,11 @@ start_open_search() { docker pull opensearchproject/opensearch:${OPEN_VERSION} docker rm ${IMAGE_OPEN_SEARCH} 2> /dev/null - docker rm ${MAIN_IMAGE_OPEN_SEARCH} 2> /dev/null + docker rm ${MAIN_CHAIN_IMAGE_OPEN_SEARCH} 2> /dev/null docker run -d --name "${IMAGE_OPEN_SEARCH}" -p 9200:9200 -p 9600:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} - docker run -d --name "${MAIN_IMAGE_OPEN_SEARCH}" -p 9201:9200 -p 9601:9600 \ + docker run -d --name "${MAIN_CHAIN_IMAGE_OPEN_SEARCH}" -p 9201:9200 -p 9601:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} From 2bf17922e6264abf5b42422bbd1372999f053e98 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 6 Feb 2025 10:49:45 +0200 Subject: [PATCH 14/17] fixes after review --- client/elasticClientCommon.go | 11 +++++++++++ factory/runType/sovereignRunTypeComponentsFactory.go | 8 +------- process/factory/indexerFactory.go | 11 +---------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/client/elasticClientCommon.go b/client/elasticClientCommon.go index 553ca048..80ab9790 100644 --- a/client/elasticClientCommon.go +++ b/client/elasticClientCommon.go @@ -6,11 +6,14 @@ import ( "fmt" "io" "io/ioutil" + "math" "net/http" "net/url" "strings" + "time" "github.com/elastic/go-elasticsearch/v7/esapi" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) @@ -268,3 +271,11 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE return nil } + +// RetryBackOff returns elastic retry backoff duration +func RetryBackOff(attempt int) time.Duration { + d := time.Duration(math.Exp2(float64(attempt))) * time.Second + log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d) + + return d +} diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 62aa1e2d..42e9fbb9 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -1,9 +1,7 @@ package runType import ( - "math" "net/http" - "time" "github.com/elastic/go-elasticsearch/v7" @@ -56,7 +54,7 @@ func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elast Password: mainChainElastic.Password, Logger: &logging.CustomLogger{}, RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, + RetryBackoff: client.RetryBackOff, } return client.NewMainChainElasticClient(argsEsClient, mainChainElastic.Enabled) } else { @@ -64,10 +62,6 @@ func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elast } } -func retryBackOff(attempt int) time.Duration { - return time.Duration(math.Exp2(float64(attempt))) * time.Second -} - // IsInterfaceNil returns true if there is no value under the interface func (srtcf *sovereignRunTypeComponentsFactory) IsInterfaceNil() bool { return srtcf == nil diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 075516a6..5def543a 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -2,9 +2,7 @@ package factory import ( "fmt" - "math" "net/http" - "time" "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core" @@ -101,13 +99,6 @@ func createManagedRunTypeComponents(factory runType.RunTypeComponentsCreator) (r return managedRunTypeComponents, nil } -func retryBackOff(attempt int) time.Duration { - d := time.Duration(math.Exp2(float64(attempt))) * time.Second - log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d) - - return d -} - func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcessor, error) { databaseClient, err := createElasticClient(args) if err != nil { @@ -141,7 +132,7 @@ func createElasticClient(args ArgsIndexerFactory) (elasticproc.DatabaseClientHan Password: args.Password, Logger: &logging.CustomLogger{}, RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, + RetryBackoff: client.RetryBackOff, } if check.IfNil(args.StatusMetrics) { From c54509b9f09c77d3d70ceb4bf53e6ae9bb50d4ad Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 10 Feb 2025 10:03:06 +0200 Subject: [PATCH 15/17] fixes after review --- client/disabled/disabledElasticClient.go | 78 +++++++++++++++++++ ..._test.go => disabledElasticClient_test.go} | 0 client/disabled/elasticClient.go | 78 ------------------- cmd/elasticindexer/config/config.toml | 1 + cmd/elasticindexer/config/prefs.toml | 2 + config/config.go | 4 +- .../tokens/sovereignIndexTokensHandler.go | 4 +- 7 files changed, 85 insertions(+), 82 deletions(-) create mode 100644 client/disabled/disabledElasticClient.go rename client/disabled/{elasticClient_test.go => disabledElasticClient_test.go} (100%) delete mode 100644 client/disabled/elasticClient.go diff --git a/client/disabled/disabledElasticClient.go b/client/disabled/disabledElasticClient.go new file mode 100644 index 00000000..313c92bf --- /dev/null +++ b/client/disabled/disabledElasticClient.go @@ -0,0 +1,78 @@ +package disabled + +import ( + "bytes" + "context" +) + +type disabledElasticClient struct{} + +// NewDisabledElasticClient - +func NewDisabledElasticClient() *disabledElasticClient { + return &disabledElasticClient{} +} + +// DoBulkRequest - +func (dec *disabledElasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { + return nil +} + +// DoQueryRemove - +func (dec *disabledElasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// DoMultiGet - +func (dec *disabledElasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { + return nil +} + +// DoScrollRequest - +func (dec *disabledElasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (dec *disabledElasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { + return 0, nil +} + +// UpdateByQuery - +func (dec *disabledElasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// PutMappings - +func (dec *disabledElasticClient) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreateIndex - +func (dec *disabledElasticClient) CheckAndCreateIndex(_ string) error { + return nil +} + +// CheckAndCreateAlias - +func (dec *disabledElasticClient) CheckAndCreateAlias(_ string, _ string) error { + return nil +} + +// CheckAndCreateTemplate - +func (dec *disabledElasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreatePolicy - +func (dec *disabledElasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { + return nil +} + +// IsEnabled - +func (dec *disabledElasticClient) IsEnabled() bool { + return false +} + +// IsInterfaceNil - returns true if there is no value under the interface +func (dec *disabledElasticClient) IsInterfaceNil() bool { + return dec == nil +} diff --git a/client/disabled/elasticClient_test.go b/client/disabled/disabledElasticClient_test.go similarity index 100% rename from client/disabled/elasticClient_test.go rename to client/disabled/disabledElasticClient_test.go diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go deleted file mode 100644 index 41ccc6c0..00000000 --- a/client/disabled/elasticClient.go +++ /dev/null @@ -1,78 +0,0 @@ -package disabled - -import ( - "bytes" - "context" -) - -type elasticClient struct{} - -// NewDisabledElasticClient - -func NewDisabledElasticClient() *elasticClient { - return &elasticClient{} -} - -// DoBulkRequest - -func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { - return nil -} - -// DoQueryRemove - -func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// DoMultiGet - -func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { - return nil -} - -// DoScrollRequest - -func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { - return nil -} - -// DoCountRequest - -func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { - return 0, nil -} - -// UpdateByQuery - -func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// PutMappings - -func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreateIndex - -func (ec *elasticClient) CheckAndCreateIndex(_ string) error { - return nil -} - -// CheckAndCreateAlias - -func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error { - return nil -} - -// CheckAndCreateTemplate - -func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreatePolicy - -func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { - return nil -} - -// IsEnabled - -func (ec *elasticClient) IsEnabled() bool { - return false -} - -// IsInterfaceNil - returns true if there is no value under the interface -func (ec *elasticClient) IsInterfaceNil() bool { - return ec == nil -} diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index b63329c7..5fdf7122 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -4,6 +4,7 @@ "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "esdts", "values", "events" ] + esdt-prefix = "" [config.address-converter] length = 32 type = "bech32" diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 42dc095f..bc6a2743 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -24,6 +24,8 @@ password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB + # Configuration for main chain elastic cluster + # Used by the sovereign chain indexer to index incoming new tokens properties [config.main-chain-elastic-cluster] enabled = true url = "http://localhost:9201" diff --git a/config/config.go b/config/config.go index 60c56b4d..203bd6e7 100644 --- a/config/config.go +++ b/config/config.go @@ -4,6 +4,7 @@ package config type Config struct { Config struct { AvailableIndices []string `toml:"available-indices"` + ESDTPrefix string `toml:"esdt-prefix"` AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` @@ -29,8 +30,7 @@ type Config struct { LogsPath string `toml:"logs-path"` } `toml:"logs"` } `toml:"config"` - Sovereign bool - ESDTPrefix string + Sovereign bool } // ClusterConfig will hold the config for the Elasticsearch cluster diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index d814f31c..1a1e1385 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -50,7 +50,7 @@ func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elas return err } - return sit.indexNewTokens(mainChainTokens.Docs, buffSlice) + return sit.serializeNewTokens(mainChainTokens.Docs, buffSlice) } func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { @@ -107,7 +107,7 @@ func getTokenCollection(hasPrefix bool, tokenIdentifier string) string { return "" } -func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { +func (sit *sovereignIndexTokensHandler) serializeNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { for _, responseToken := range responseTokensInfo { token, identifier := formatToken(responseToken) From 5cec770a51a610f4e3bcae322b9e4d987e10c893 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 10 Feb 2025 10:23:42 +0200 Subject: [PATCH 16/17] fixes after review --- client/disabled/disabledElasticClient.go | 78 ------------------- client/disabled/elasticClient.go | 78 +++++++++++++++++++ ...icClient_test.go => elasticClient_test.go} | 0 factory/runType/runTypeComponentsHandler.go | 3 +- process/dataindexer/errors.go | 3 + process/elasticproc/check.go | 3 +- ...ndler.go => disabledIndexTokensHandler.go} | 12 +-- ....go => disabledIndexTokensHandler_test.go} | 0 process/elasticproc/transactions/errors.go | 3 - 9 files changed, 90 insertions(+), 90 deletions(-) delete mode 100644 client/disabled/disabledElasticClient.go create mode 100644 client/disabled/elasticClient.go rename client/disabled/{disabledElasticClient_test.go => elasticClient_test.go} (100%) rename process/elasticproc/tokens/{indexTokensHandler.go => disabledIndexTokensHandler.go} (50%) rename process/elasticproc/tokens/{indexTokensHandler_test.go => disabledIndexTokensHandler_test.go} (100%) diff --git a/client/disabled/disabledElasticClient.go b/client/disabled/disabledElasticClient.go deleted file mode 100644 index 313c92bf..00000000 --- a/client/disabled/disabledElasticClient.go +++ /dev/null @@ -1,78 +0,0 @@ -package disabled - -import ( - "bytes" - "context" -) - -type disabledElasticClient struct{} - -// NewDisabledElasticClient - -func NewDisabledElasticClient() *disabledElasticClient { - return &disabledElasticClient{} -} - -// DoBulkRequest - -func (dec *disabledElasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { - return nil -} - -// DoQueryRemove - -func (dec *disabledElasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// DoMultiGet - -func (dec *disabledElasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { - return nil -} - -// DoScrollRequest - -func (dec *disabledElasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { - return nil -} - -// DoCountRequest - -func (dec *disabledElasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { - return 0, nil -} - -// UpdateByQuery - -func (dec *disabledElasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// PutMappings - -func (dec *disabledElasticClient) PutMappings(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreateIndex - -func (dec *disabledElasticClient) CheckAndCreateIndex(_ string) error { - return nil -} - -// CheckAndCreateAlias - -func (dec *disabledElasticClient) CheckAndCreateAlias(_ string, _ string) error { - return nil -} - -// CheckAndCreateTemplate - -func (dec *disabledElasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreatePolicy - -func (dec *disabledElasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { - return nil -} - -// IsEnabled - -func (dec *disabledElasticClient) IsEnabled() bool { - return false -} - -// IsInterfaceNil - returns true if there is no value under the interface -func (dec *disabledElasticClient) IsInterfaceNil() bool { - return dec == nil -} diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go new file mode 100644 index 00000000..41ccc6c0 --- /dev/null +++ b/client/disabled/elasticClient.go @@ -0,0 +1,78 @@ +package disabled + +import ( + "bytes" + "context" +) + +type elasticClient struct{} + +// NewDisabledElasticClient - +func NewDisabledElasticClient() *elasticClient { + return &elasticClient{} +} + +// DoBulkRequest - +func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { + return nil +} + +// DoQueryRemove - +func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// DoMultiGet - +func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { + return nil +} + +// DoScrollRequest - +func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { + return 0, nil +} + +// UpdateByQuery - +func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// PutMappings - +func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreateIndex - +func (ec *elasticClient) CheckAndCreateIndex(_ string) error { + return nil +} + +// CheckAndCreateAlias - +func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error { + return nil +} + +// CheckAndCreateTemplate - +func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreatePolicy - +func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { + return nil +} + +// IsEnabled - +func (ec *elasticClient) IsEnabled() bool { + return false +} + +// IsInterfaceNil - returns true if there is no value under the interface +func (ec *elasticClient) IsInterfaceNil() bool { + return ec == nil +} diff --git a/client/disabled/disabledElasticClient_test.go b/client/disabled/elasticClient_test.go similarity index 100% rename from client/disabled/disabledElasticClient_test.go rename to client/disabled/elasticClient_test.go diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index b9697008..113be479 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" + elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) @@ -80,7 +81,7 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { return transactions.ErrNilRewardTxDataHandler } if check.IfNil(mrtc.indexTokensHandler) { - return transactions.ErrNilIndexTokensHandler + return elasticIndexer.ErrNilIndexTokensHandler } return nil } diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index 9b4fbd0f..9495e93f 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -88,3 +88,6 @@ var ErrNilOperationsHandler = errors.New("nil operations handler") // ErrNilBlockContainerHandler signals that a nil block container handler has been provided var ErrNilBlockContainerHandler = errors.New("nil bock container handler") + +// ErrNilIndexTokensHandler signals that a nil index tokens handler has been provided +var ErrNilIndexTokensHandler = errors.New("nil index tokens handler") diff --git a/process/elasticproc/check.go b/process/elasticproc/check.go index b3953d15..d15a19fc 100644 --- a/process/elasticproc/check.go +++ b/process/elasticproc/check.go @@ -4,7 +4,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) func checkArguments(arguments *ArgElasticProcessor) error { @@ -42,7 +41,7 @@ func checkArguments(arguments *ArgElasticProcessor) error { return elasticIndexer.ErrNilOperationsHandler } if check.IfNilReflect(arguments.IndexTokensHandler) { - return transactions.ErrNilIndexTokensHandler + return elasticIndexer.ErrNilIndexTokensHandler } return nil diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/disabledIndexTokensHandler.go similarity index 50% rename from process/elasticproc/tokens/indexTokensHandler.go rename to process/elasticproc/tokens/disabledIndexTokensHandler.go index 99a4b8c4..18fc63f8 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/disabledIndexTokensHandler.go @@ -5,19 +5,19 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) -type indexTokensHandler struct{} +type disabledTndexTokensHandler struct{} // NewDisabledIndexTokensHandler creates a new disabled index tokens handler -func NewDisabledIndexTokensHandler() *indexTokensHandler { - return &indexTokensHandler{} +func NewDisabledIndexTokensHandler() *disabledTndexTokensHandler { + return &disabledTndexTokensHandler{} } // IndexCrossChainTokens should do nothing and return no error -func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { +func (dit *disabledTndexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { return nil } // IsInterfaceNil returns true if there is no value under the interface -func (it *indexTokensHandler) IsInterfaceNil() bool { - return it == nil +func (dit *disabledTndexTokensHandler) IsInterfaceNil() bool { + return dit == nil } diff --git a/process/elasticproc/tokens/indexTokensHandler_test.go b/process/elasticproc/tokens/disabledIndexTokensHandler_test.go similarity index 100% rename from process/elasticproc/tokens/indexTokensHandler_test.go rename to process/elasticproc/tokens/disabledIndexTokensHandler_test.go diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go index f9280c86..9f91b390 100644 --- a/process/elasticproc/transactions/errors.go +++ b/process/elasticproc/transactions/errors.go @@ -9,6 +9,3 @@ var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") // ErrNilRewardTxDataHandler signals that a nil rewards tx data handler has been provided var ErrNilRewardTxDataHandler = errors.New("nil reward tx data handler") - -// ErrNilIndexTokensHandler signals that a nil index tokens handler has been provided -var ErrNilIndexTokensHandler = errors.New("nil index tokens handler") From 5978925e96504dd23120bc540c1b9c8dcd2e6ed4 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 10 Feb 2025 16:00:06 +0200 Subject: [PATCH 17/17] sovereign elastic indexer docker fixes --- .github/workflows/deploy-docker.yml | 10 ++++++---- Dockerfile-sovereign | 26 ++++++++++++++++++++++++++ Makefile | 7 +++++++ 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 Dockerfile-sovereign diff --git a/.github/workflows/deploy-docker.yml b/.github/workflows/deploy-docker.yml index df006e3f..c6d85c6b 100644 --- a/.github/workflows/deploy-docker.yml +++ b/.github/workflows/deploy-docker.yml @@ -3,6 +3,7 @@ name: Publish Docker image on: release: types: [published] + pull_request: jobs: push_to_registry: @@ -31,24 +32,25 @@ jobs: fi - name: Log in to Docker Hub + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: - images: multiversx/elastic-indexer + images: multiversx/sovereign-elastic-indexer - name: Build and push Docker image id: push uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 with: context: . - file: ./Dockerfile - push: true + file: ./Dockerfile-sovereign + push: ${{ github.event_name == 'release' && github.event.action == 'published' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - diff --git a/Dockerfile-sovereign b/Dockerfile-sovereign new file mode 100644 index 00000000..c8b72a1a --- /dev/null +++ b/Dockerfile-sovereign @@ -0,0 +1,26 @@ +FROM golang:1.20.7 as builder + +RUN apt-get update && apt-get install -y + +WORKDIR /multiversx +COPY . . + +WORKDIR /multiversx/cmd/elasticindexer + +RUN go build -o elasticindexer + +# ===== SECOND STAGE ====== +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y + +RUN useradd -m -u 1000 appuser +USER appuser + +COPY --from=builder --chown=appuser /multiversx/cmd/elasticindexer /multiversx + +EXPOSE 22111 + +WORKDIR /multiversx + +ENTRYPOINT ["./elasticindexer", "--sovereign"] +CMD ["--log-level", "*:DEBUG"] diff --git a/Makefile b/Makefile index a000f850..075cc715 100644 --- a/Makefile +++ b/Makefile @@ -36,9 +36,16 @@ integration-tests-open-search: INDEXER_IMAGE_NAME="elasticindexer" INDEXER_IMAGE_TAG="latest" DOCKER_FILE=Dockerfile +SOVEREIGN_DOCKER_FILE=Dockerfile-sovereign docker-build: docker build \ -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \ -f ${DOCKER_FILE} \ . + +docker-sovereign-build: + docker build \ + -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \ + -f ${SOVEREIGN_DOCKER_FILE} \ + .