From 5aab174a295a1e245376f2d0e96a40011e335fb5 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Tue, 1 Feb 2022 14:01:01 +0200 Subject: [PATCH 01/96] adapt indexer for new core altered accounts --- data/account.go | 6 +- dataIndexer.go | 2 +- errors.go | 6 - factory/indexerFactory.go | 5 - factory/indexerFactory_test.go | 10 - go.mod | 2 +- go.sum | 3 +- integrationtests/esdtTransfer_test.go | 5 +- integrationtests/issueToken_test.go | 7 +- .../nftTransferCrossShard_test.go | 21 +- .../nftTransferCrossWithScCall_test.go | 5 +- integrationtests/relayedTx_test.go | 12 +- integrationtests/scCallIntraShard_test.go | 10 +- integrationtests/transactions_test.go | 5 +- integrationtests/utils.go | 2 - interface.go | 4 +- mock/dbAccountsHandlerStub.go | 3 +- mock/elasticProcessorStub.go | 6 +- nilIndexer.go | 2 +- process/accounts/accountsProcessor.go | 99 ++---- process/accounts/accountsProcessor_test.go | 335 +++++++----------- process/elasticProcessor.go | 6 +- process/elasticProcessor_test.go | 10 +- process/factory/elasticProcessorFactory.go | 2 - .../factory/elasticProcessorFactory_test.go | 1 - process/interface.go | 2 +- workItems/interface.go | 2 +- workItems/workItemAccounts.go | 10 +- workItems/workItemAccounts_test.go | 6 +- workItems/workItemBlock.go | 2 +- workItems/workItemBlock_test.go | 6 +- 31 files changed, 227 insertions(+), 370 deletions(-) diff --git a/data/account.go b/data/account.go index f9b8934d..edd2b312 100644 --- a/data/account.go +++ b/data/account.go @@ -3,7 +3,7 @@ package data import ( "time" - coreData "github.com/ElrondNetwork/elrond-go-core/data" + "github.com/ElrondNetwork/elrond-go-core/data/indexer" ) // AccountInfo holds (serializable) data about an account @@ -51,13 +51,13 @@ type AccountBalanceHistory struct { // Account is a structure that is needed for regular accounts type Account struct { - UserAccount coreData.UserAccountHandler + UserAccount *indexer.AlteredAccount IsSender bool } // AccountESDT is a structure that is needed for ESDT accounts type AccountESDT struct { - Account coreData.UserAccountHandler + Account *indexer.AlteredAccount TokenIdentifier string NFTNonce uint64 IsSender bool diff --git a/dataIndexer.go b/dataIndexer.go index 5cbb0d89..d6a78a1b 100644 --- a/dataIndexer.go +++ b/dataIndexer.go @@ -140,7 +140,7 @@ func (di *dataIndexer) SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]by } // SaveAccounts will save the provided accounts -func (di *dataIndexer) SaveAccounts(timestamp uint64, accounts []coreData.UserAccountHandler) error { +func (di *dataIndexer) SaveAccounts(timestamp uint64, accounts map[string]*indexer.AlteredAccount) error { wi := workItems.NewItemAccounts(di.elasticProcessor, timestamp, accounts) di.dispatcher.Add(wi) diff --git a/errors.go b/errors.go index 68c9494c..bd567e9a 100644 --- a/errors.go +++ b/errors.go @@ -8,9 +8,6 @@ var ErrNilMarshalizer = errors.New("nil marshalizer provided") // ErrNilPubkeyConverter signals that an operation has been attempted to or with a nil public key converter implementation var ErrNilPubkeyConverter = errors.New("nil pubkey converter") -// ErrNilAccountsDB signals that a nil accounts database has been provided -var ErrNilAccountsDB = errors.New("nil accounts db") - // ErrNegativeDenominationValue signals that a negative denomination value has been provided var ErrNegativeDenominationValue = errors.New("negative denomination value") @@ -47,9 +44,6 @@ var ErrNilHasher = errors.New("nil hasher provided") // ErrNilUrl signals that the provided url is empty var ErrNilUrl = errors.New("url is empty") -// ErrCannotCastAccountHandlerToUserAccount signals that cast from AccountHandler to UserAccountHandler was not OK -var ErrCannotCastAccountHandlerToUserAccount = errors.New("cannot cast AccountHandler to UserAccountHandler") - // ErrNilHeaderHandler signals that a nil header handler has been provided var ErrNilHeaderHandler = errors.New("nil header handler") diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index dffed650..0b311fb6 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -32,7 +32,6 @@ type ArgsIndexerFactory struct { Hasher hashing.Hasher AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter - AccountsDB indexer.AccountsAdapter TransactionFeeCalculator indexer.FeesProcessorHandler } @@ -87,7 +86,6 @@ func createElasticProcessor(args *ArgsIndexerFactory) (indexer.ElasticProcessor, ValidatorPubkeyConverter: args.ValidatorPubkeyConverter, UseKibana: args.UseKibana, DBClient: databaseClient, - AccountsDB: args.AccountsDB, Denomination: args.Denomination, TransactionFeeCalculator: args.TransactionFeeCalculator, IsInImportDBMode: args.IsInImportDBMode, @@ -120,9 +118,6 @@ func checkDataIndexerParams(arguments *ArgsIndexerFactory) error { if check.IfNil(arguments.TransactionFeeCalculator) { return indexer.ErrNilTransactionFeeCalculator } - if check.IfNil(arguments.AccountsDB) { - return indexer.ErrNilAccountsDB - } if check.IfNil(arguments.ShardCoordinator) { return indexer.ErrNilShardCoordinator } diff --git a/factory/indexerFactory_test.go b/factory/indexerFactory_test.go index 980f5487..f1f8bf99 100644 --- a/factory/indexerFactory_test.go +++ b/factory/indexerFactory_test.go @@ -26,7 +26,6 @@ func createMockIndexerFactoryArgs() *ArgsIndexerFactory { ValidatorPubkeyConverter: &mock.PubkeyConverterMock{}, TemplatesPath: "../testdata", EnabledIndexes: []string{"blocks", "transactions", "miniblocks", "validators", "round", "accounts", "rating"}, - AccountsDB: &mock.AccountsStub{}, TransactionFeeCalculator: &mock.EconomicsHandlerStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, IsInImportDBMode: false, @@ -84,15 +83,6 @@ func TestNewIndexerFactory(t *testing.T) { }, exError: indexer.ErrNilHasher, }, - { - name: "NilAccountsDB", - argsFunc: func() *ArgsIndexerFactory { - args := createMockIndexerFactoryArgs() - args.AccountsDB = nil - return args - }, - exError: indexer.ErrNilAccountsDB, - }, { name: "EmptyUrl", argsFunc: func() *ArgsIndexerFactory { diff --git a/go.mod b/go.mod index 7a560c6e..4375b9e3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.15 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.7 + github.com/ElrondNetwork/elrond-go-core v1.1.8-0.20220128143134-2fee54561061 github.com/ElrondNetwork/elrond-go-logger v1.0.5 github.com/ElrondNetwork/elrond-vm-common v1.2.9 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index e30613d2..09c42a48 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= -github.com/ElrondNetwork/elrond-go-core v1.1.7 h1:pusBmtsS7lUzrDovW7XKWwhXIt3xRQg5hsB5bV8GkZQ= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= +github.com/ElrondNetwork/elrond-go-core v1.1.8-0.20220128143134-2fee54561061 h1:+qjz6/oMTVU2sp2MCUYpTvxpT63Dgm8jHDboqkEhTGQ= +github.com/ElrondNetwork/elrond-go-core v1.1.8-0.20220128143134-2fee54561061/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.5 h1:tB/HBvV9IVeCaSrGakX+GLGu7K5UPLv8gA0TNKPOTOU= github.com/ElrondNetwork/elrond-go-logger v1.0.5/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index fa512e1a..7bdbb307 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -27,11 +27,10 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("esdtTransfer") @@ -98,7 +97,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { string(scrHash1): scr1, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 640dc154..40a6b377 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -26,13 +26,12 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) body := &dataBlock.Body{} @@ -59,7 +58,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -88,7 +87,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids = []string{"SEMI-abcd"} diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 08d0e797..d78e1999 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -36,11 +36,10 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("nftTransferWithSCCall") @@ -94,7 +93,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { string(scrHash1): scr1, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -131,7 +130,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -149,11 +148,10 @@ func TestNFTTransferCrossShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("nftTransfer") @@ -217,7 +215,7 @@ func TestNFTTransferCrossShard(t *testing.T) { string(scrHash2): scr2, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -252,7 +250,7 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -269,11 +267,10 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("nftTransferCross") @@ -321,7 +318,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -375,7 +372,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t string(scrHash2): scr2, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index f5596825..0a9514b2 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -27,11 +27,10 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("nftTransferWithScCall") @@ -84,7 +83,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { string(scrHash2): scr2, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 2f0ef6e0..10d01f2a 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -30,11 +30,10 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("relayedTx") @@ -90,7 +89,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { string(scrHash1): scr1, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -129,7 +128,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -146,11 +145,10 @@ func TestRelayedTransactionIntraShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("relayedTxIntra") @@ -219,7 +217,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { string(scrHash2): scr2, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 7b9684d3..fd23fe16 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -28,11 +28,10 @@ func TestTransactionWithSCCallFail(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("t") @@ -79,7 +78,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -98,11 +97,10 @@ func TestTransactionWithScCallSuccess(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("txHashClaimRewards") @@ -148,7 +146,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index cd5cb669..07941a94 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -24,11 +24,10 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) txHash := []byte("hash") @@ -59,7 +58,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 9092186c..a7d18555 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -28,7 +28,6 @@ func createESClient(url string) (process.DatabaseClientHandler, error) { // CreateElasticProcessor - func CreateElasticProcessor( esClient process.DatabaseClientHandler, - accountsDB indexer.AccountsAdapter, shardCoordinator indexer.ShardCoordinator, feeProcessor indexer.FeesProcessorHandler, ) (indexer.ElasticProcessor, error) { @@ -38,7 +37,6 @@ func CreateElasticProcessor( AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32), DBClient: esClient, - AccountsDB: accountsDB, ShardCoordinator: shardCoordinator, TransactionFeeCalculator: feeProcessor, EnabledIndexes: []string{indexer.TransactionsIndex, indexer.LogsIndex, indexer.AccountsESDTIndex, indexer.ScResultsIndex, indexer.ReceiptsIndex, indexer.BlockIndex, indexer.AccountsIndex, indexer.TokensIndex}, diff --git a/interface.go b/interface.go index a2cfc8c4..3d74c415 100644 --- a/interface.go +++ b/interface.go @@ -33,7 +33,7 @@ type ElasticProcessor interface { RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error SaveRoundsInfo(infos []*data.RoundInfo) error SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error @@ -64,7 +64,7 @@ type Indexer interface { SaveRoundsInfo(roundsInfos []*indexer.RoundInfo) error SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error SaveValidatorsRating(indexID string, infoRating []*indexer.ValidatorRatingInfo) error - SaveAccounts(blockTimestamp uint64, acc []coreData.UserAccountHandler) error + SaveAccounts(blockTimestamp uint64, acc map[string]*indexer.AlteredAccount) error FinalizedBlock(headerHash []byte) error Close() error IsInterfaceNil() bool diff --git a/mock/dbAccountsHandlerStub.go b/mock/dbAccountsHandlerStub.go index e28d9d55..76caef30 100644 --- a/mock/dbAccountsHandlerStub.go +++ b/mock/dbAccountsHandlerStub.go @@ -4,6 +4,7 @@ import ( "bytes" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elrond-go-core/data/indexer" ) // DBAccountsHandlerStub - @@ -13,7 +14,7 @@ type DBAccountsHandlerStub struct { } // GetAccounts - -func (dba *DBAccountsHandlerStub) GetAccounts(_ data.AlteredAccountsHandler) ([]*data.Account, []*data.AccountESDT) { +func (dba *DBAccountsHandlerStub) GetAccounts(_ data.AlteredAccountsHandler, _ map[string]*indexer.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { return nil, nil } diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index a85ba636..74edba1b 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -21,7 +21,7 @@ type ElasticProcessorStub struct { RemoveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error RemoveTransactionsCalled func(header coreData.HeaderHandler, body *block.Body) error SaveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error - SaveTransactionsCalled func(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) error + SaveTransactionsCalled func(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error SaveValidatorsRatingCalled func(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error SaveRoundsInfoCalled func(infos []*data.RoundInfo) error SaveShardValidatorsPubKeysCalled func(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error @@ -75,9 +75,9 @@ func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, b } // SaveTransactions - -func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) error { +func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { if eim.SaveTransactionsCalled != nil { - return eim.SaveTransactionsCalled(body, header, pool) + return eim.SaveTransactionsCalled(body, header, pool, coreAlteredAccounts) } return nil } diff --git a/nilIndexer.go b/nilIndexer.go index 7de641d9..afb6ee36 100644 --- a/nilIndexer.go +++ b/nilIndexer.go @@ -40,7 +40,7 @@ func (ni *NilIndexer) SaveValidatorsPubKeys(_ map[uint32][][]byte, _ uint32) err } // SaveAccounts returns nil -func (ni *NilIndexer) SaveAccounts(_ uint64, _ []data.UserAccountHandler) error { +func (ni *NilIndexer) SaveAccounts(_ uint64, _ map[string]*indexer.AlteredAccount) error { return nil } diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index 996cfa88..c0183500 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -11,8 +11,8 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" - coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/esdt" + coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" ) @@ -23,7 +23,6 @@ var log = logger.GetOrCreate("indexer/process/accounts") type accountsProcessor struct { internalMarshalizer marshal.Marshalizer addressPubkeyConverter core.PubkeyConverter - accountsDB indexer.AccountsAdapter balanceConverter indexer.BalanceConverter } @@ -31,7 +30,6 @@ type accountsProcessor struct { func NewAccountsProcessor( marshalizer marshal.Marshalizer, addressPubkeyConverter core.PubkeyConverter, - accountsDB indexer.AccountsAdapter, balanceConverter indexer.BalanceConverter, ) (*accountsProcessor, error) { if check.IfNil(marshalizer) { @@ -40,9 +38,6 @@ func NewAccountsProcessor( if check.IfNil(addressPubkeyConverter) { return nil, indexer.ErrNilPubkeyConverter } - if check.IfNil(accountsDB) { - return nil, indexer.ErrNilAccountsDB - } if check.IfNil(balanceConverter) { return nil, indexer.ErrNilBalanceConverter } @@ -50,13 +45,12 @@ func NewAccountsProcessor( return &accountsProcessor{ internalMarshalizer: marshalizer, addressPubkeyConverter: addressPubkeyConverter, - accountsDB: accountsDB, balanceConverter: balanceConverter, }, nil } // GetAccounts will get accounts for regular operations and esdt operations -func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHandler) ([]*data.Account, []*data.AccountESDT) { +func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) @@ -66,13 +60,12 @@ func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHan allAlteredAccounts := alteredAccounts.GetAll() for address, altered := range allAlteredAccounts { - userAccount, err := ap.getUserAccount(address) - if err != nil || check.IfNil(userAccount) { - log.Warn("cannot get user account", "address", address, "error", err) + alteredAccount := coreAlteredAccounts[address] + if alteredAccount == nil { continue } - regularAccounts, esdtAccounts := splitAlteredAccounts(userAccount, altered) + regularAccounts, esdtAccounts := splitAlteredAccounts(alteredAccount, altered) regularAccountsToIndex = append(regularAccountsToIndex, regularAccounts...) accountsToIndexESDT = append(accountsToIndexESDT, esdtAccounts...) @@ -81,13 +74,15 @@ func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHan return regularAccountsToIndex, accountsToIndexESDT } -func splitAlteredAccounts(userAccount coreData.UserAccountHandler, altered []*data.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { +func splitAlteredAccounts( + account *coreIndexerData.AlteredAccount, + altered []*data.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) for _, info := range altered { if info.IsESDTOperation || info.IsNFTOperation { accountsToIndexESDT = append(accountsToIndexESDT, &data.AccountESDT{ - Account: userAccount, + Account: account, TokenIdentifier: info.TokenIdentifier, IsSender: info.IsSender, IsNFTOperation: info.IsNFTOperation, @@ -97,13 +92,13 @@ func splitAlteredAccounts(userAccount coreData.UserAccountHandler, altered []*da } // if the balance of the ESDT receiver is 0 the receiver is a new account most probably, and we should index it - ignoreReceiver := !info.BalanceChange && notZeroBalance(userAccount) && !info.IsSender + ignoreReceiver := !info.BalanceChange && notZeroBalance(account.Balance) && !info.IsSender if ignoreReceiver { continue } regularAccountsToIndex = append(regularAccountsToIndex, &data.Account{ - UserAccount: userAccount, + UserAccount: account, IsSender: info.IsSender, }) } @@ -111,47 +106,29 @@ func splitAlteredAccounts(userAccount coreData.UserAccountHandler, altered []*da return regularAccountsToIndex, accountsToIndexESDT } -func notZeroBalance(userAccount coreData.UserAccountHandler) bool { - if userAccount.GetBalance() == nil { - return false - } - - return userAccount.GetBalance().Cmp(big.NewInt(0)) > 0 -} - -func (ap *accountsProcessor) getUserAccount(address string) (coreData.UserAccountHandler, error) { - addressBytes, err := ap.addressPubkeyConverter.Decode(address) - if err != nil { - return nil, err - } - - account, err := ap.accountsDB.LoadAccount(addressBytes) - if err != nil { - return nil, err - } - - userAccount, ok := account.(coreData.UserAccountHandler) - if !ok { - return nil, indexer.ErrCannotCastAccountHandlerToUserAccount - } - - return userAccount, nil +func notZeroBalance(balance string) bool { + return len(balance) > 0 && balance != "0" } // PrepareRegularAccountsMap will prepare a map of regular accounts func (ap *accountsProcessor) PrepareRegularAccountsMap(accounts []*data.Account) map[string]*data.AccountInfo { accountsMap := make(map[string]*data.AccountInfo) for _, userAccount := range accounts { - address := ap.addressPubkeyConverter.Encode(userAccount.UserAccount.AddressBytes()) - balance := userAccount.UserAccount.GetBalance() + address := userAccount.UserAccount.Address + addressBytes, err := ap.addressPubkeyConverter.Decode(address) + if err != nil { + log.Warn("PrepareRegularAccountsMap: cannot decode address", "address", address, "error", err) + continue + } + balance, _ := big.NewInt(0).SetString(userAccount.UserAccount.Balance, 10) balanceAsFloat := ap.balanceConverter.ComputeBalanceAsFloat(balance) acc := &data.AccountInfo{ Address: address, - Nonce: userAccount.UserAccount.GetNonce(), + Nonce: userAccount.UserAccount.Nonce, Balance: balance.String(), BalanceNum: balanceAsFloat, IsSender: userAccount.IsSender, - IsSmartContract: core.IsSmartContractAddress(userAccount.UserAccount.AddressBytes()), + IsSmartContract: core.IsSmartContractAddress(addressBytes), TotalBalanceWithStake: balance.String(), TotalBalanceWithStakeNum: balanceAsFloat, } @@ -168,7 +145,12 @@ func (ap *accountsProcessor) PrepareAccountsMapESDT( ) map[string]*data.AccountInfo { accountsESDTMap := make(map[string]*data.AccountInfo) for _, accountESDT := range accounts { - address := ap.addressPubkeyConverter.Encode(accountESDT.Account.AddressBytes()) + address := accountESDT.Account.Address + addressBytes, err := ap.addressPubkeyConverter.Decode(address) + if err != nil { + log.Warn("PrepareAccountsMapESDT: cannot decode address", "address", address, "error", err) + continue + } balance, properties, tokenMetaData, err := ap.getESDTInfo(accountESDT) if err != nil { log.Warn("cannot get esdt info from account", @@ -186,7 +168,7 @@ func (ap *accountsProcessor) PrepareAccountsMapESDT( BalanceNum: ap.balanceConverter.ComputeESDTBalanceAsFloat(balance), Properties: properties, IsSender: accountESDT.IsSender, - IsSmartContract: core.IsSmartContractAddress(accountESDT.Account.AddressBytes()), + IsSmartContract: core.IsSmartContractAddress(addressBytes), Data: tokenMetaData, } @@ -229,21 +211,16 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In return big.NewInt(0), "", nil, nil } - tokenKey := []byte(core.ElrondProtectedKeyPrefix + core.ESDTKeyIdentifier + accountESDT.TokenIdentifier) - if accountESDT.IsNFTOperation { - nonceBig := big.NewInt(0).SetUint64(accountESDT.NFTNonce) - tokenKey = append(tokenKey, nonceBig.Bytes()...) - } - - valueBytes, err := accountESDT.Account.RetrieveValueFromDataTrieTracker(tokenKey) - if err != nil { - return nil, "", nil, err - } - esdtToken := &esdt.ESDigitalToken{} - err = ap.internalMarshalizer.Unmarshal(esdtToken, valueBytes) - if err != nil { - return nil, "", nil, err + for _, tokenData := range accountESDT.Account.Tokens { + if tokenData.Identifier == accountESDT.TokenIdentifier && tokenData.Nonce == accountESDT.NFTNonce { + value, _ := big.NewInt(0).SetString(tokenData.Balance, 10) + esdtToken = &esdt.ESDigitalToken{ + Value: value, + Properties: []byte(tokenData.Properties), + TokenMetaData: tokenData.MetaData, + } + } } if esdtToken.Value == nil { diff --git a/process/accounts/accountsProcessor_test.go b/process/accounts/accountsProcessor_test.go index 06bc7d03..59c0a18a 100644 --- a/process/accounts/accountsProcessor_test.go +++ b/process/accounts/accountsProcessor_test.go @@ -1,8 +1,8 @@ package accounts import ( + "bytes" "encoding/hex" - "encoding/json" "errors" "math/big" "testing" @@ -13,9 +13,8 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/esdt" + coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/marshal" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,41 +25,34 @@ func TestNewAccountsProcessor(t *testing.T) { tests := []struct { name string - argsFunc func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) + argsFunc func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) exError error }{ { name: "NilBalanceConverter", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, &mock.AccountsStub{}, nil + argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { + return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, nil }, exError: indexer.ErrNilBalanceConverter, }, { name: "NilMarshalizer", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return nil, &mock.PubkeyConverterMock{}, &mock.AccountsStub{}, balanceConverter + argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { + return nil, &mock.PubkeyConverterMock{}, balanceConverter }, exError: indexer.ErrNilMarshalizer, }, { name: "NilPubKeyConverter", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, nil, &mock.AccountsStub{}, balanceConverter + argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { + return &mock.MarshalizerMock{}, nil, balanceConverter }, exError: indexer.ErrNilPubkeyConverter, }, - { - name: "NilAccounts", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, nil, balanceConverter - }, - exError: indexer.ErrNilAccountsDB, - }, { name: "ShouldWork", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, &mock.AccountsStub{}, balanceConverter + argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { + return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, balanceConverter }, exError: nil, }, @@ -77,9 +69,9 @@ func TestNewAccountsProcessor(t *testing.T) { func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), &mock.AccountsStub{}, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) - regularAccounts, esdtAccounts := ap.GetAccounts(nil) + regularAccounts, esdtAccounts := ap.GetAccounts(nil, nil) require.Len(t, regularAccounts, 0) require.Len(t, esdtAccounts, 0) } @@ -87,47 +79,30 @@ func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { func TestAccountsProcessor_PrepareRegularAccountsMapWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), &mock.AccountsStub{}, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) accountsInfo := ap.PrepareRegularAccountsMap(nil) require.Len(t, accountsInfo, 0) } -func TestGetESDTInfo_CannotRetriveValueShoudError(t *testing.T) { - t.Parallel() - - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), &mock.AccountsStub{}, balanceConverter) - require.NotNil(t, ap) - - localErr := errors.New("local error") - wrapAccount := &data.AccountESDT{ - Account: &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return nil, localErr - }, - }, - TokenIdentifier: "token", - } - _, _, _, err := ap.getESDTInfo(wrapAccount) - require.Equal(t, localErr, err) -} - func TestGetESDTInfo(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), &mock.AccountsStub{}, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - Properties: []byte("ok"), - } - tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) + Account: &coreIndexerData.AlteredAccount{ + Address: "", + Balance: "1000", + Nonce: 0, + Tokens: []*coreIndexerData.AccountTokenData{ + { + Identifier: tokenIdentifier, + Balance: "1000", + Properties: "ok", + }, }, }, TokenIdentifier: tokenIdentifier, @@ -141,20 +116,22 @@ func TestGetESDTInfo(t *testing.T) { func TestGetESDTInfoNFT(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), &mock.AccountsStub{}, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1), - Properties: []byte("ok"), - } - tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - assert.Equal(t, append([]byte("ELRONDesdttoken-001"), 0xa), key) - return json.Marshal(esdtToken) + Account: &coreIndexerData.AlteredAccount{ + Address: "", + Balance: "1", + Nonce: 10, + Tokens: []*coreIndexerData.AccountTokenData{ + { + Identifier: tokenIdentifier, + Balance: "1", + Nonce: 10, + Properties: "ok", + }, }, }, TokenIdentifier: tokenIdentifier, @@ -171,28 +148,31 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { t.Parallel() pubKeyConverter := mock.NewPubkeyConverterMock(32) - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, pubKeyConverter, &mock.AccountsStub{}, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, pubKeyConverter, balanceConverter) require.NotNil(t, ap) nftName := "Test-nft" creator := []byte("010101") - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1), - Properties: []byte("ok"), - TokenMetaData: &esdt.MetaData{ - Nonce: 1, - Name: []byte(nftName), - Creator: creator, - Royalties: 2, - }, - } tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - assert.Equal(t, append([]byte("ELRONDesdttoken-001"), 0xa), key) - return json.Marshal(esdtToken) + Account: &coreIndexerData.AlteredAccount{ + Address: "", + Balance: "1", + Nonce: 1, + Tokens: []*coreIndexerData.AccountTokenData{ + { + Identifier: tokenIdentifier, + Balance: "1", + Properties: "ok", + Nonce: 10, + MetaData: &esdt.MetaData{ + Nonce: 10, + Name: []byte(nftName), + Creator: creator, + Royalties: 2, + }, + }, }, }, TokenIdentifier: tokenIdentifier, @@ -214,13 +194,11 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { t.Parallel() addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{} - accountsStub := &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, + acc := &coreIndexerData.AlteredAccount{} + alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ + addr: acc, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), accountsStub, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -229,10 +207,12 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { TokenIdentifier: "", }) - accounts, esdtAccounts := ap.GetAccounts(alteredAccounts) + accounts, esdtAccounts := ap.GetAccounts(alteredAccounts, alteredAccountsMap) require.Equal(t, 0, len(esdtAccounts)) require.Equal(t, []*data.Account{ - {UserAccount: mockAccount}, + { + UserAccount: acc, + }, }, accounts) } @@ -240,17 +220,14 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { t.Parallel() addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - GetBalanceCalled: func() *big.Int { - return big.NewInt(1) - }, + acc := &coreIndexerData.AlteredAccount{ + Address: addr, + Balance: "1", } - accountsStub := &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, + alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ + addr: acc, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), accountsStub, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -258,10 +235,10 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { IsESDTOperation: true, TokenIdentifier: "token", }) - accounts, esdtAccounts := ap.GetAccounts(alteredAccounts) + accounts, esdtAccounts := ap.GetAccounts(alteredAccounts, alteredAccountsMap) require.Equal(t, 0, len(accounts)) require.Equal(t, []*data.AccountESDT{ - {Account: mockAccount, TokenIdentifier: "token"}, + {Account: acc, TokenIdentifier: "token"}, }, esdtAccounts) } @@ -269,17 +246,11 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco t.Parallel() addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - GetBalanceCalled: func() *big.Int { - return big.NewInt(0) - }, - } - accountsStub := &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, + acc := &coreIndexerData.AlteredAccount{} + alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ + addr: acc, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), accountsStub, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -287,91 +258,84 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco IsESDTOperation: true, TokenIdentifier: "token", }) - accounts, esdtAccounts := ap.GetAccounts(alteredAccounts) + accounts, esdtAccounts := ap.GetAccounts(alteredAccounts, alteredAccountsMap) require.Equal(t, 1, len(accounts)) require.Equal(t, []*data.AccountESDT{ - {Account: mockAccount, TokenIdentifier: "token"}, + {Account: acc, TokenIdentifier: "token"}, }, esdtAccounts) require.Equal(t, []*data.Account{ - {UserAccount: mockAccount, IsSender: false}, + {UserAccount: acc, IsSender: false}, }, accounts) } func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { t.Parallel() - addr := string(make([]byte, 32)) - mockAccount := &mock.UserAccountStub{ - GetNonceCalled: func() uint64 { - return 1 - }, - GetBalanceCalled: func() *big.Int { - return big.NewInt(1000) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, + addrBytes := bytes.Repeat([]byte{0}, 32) + addr := hex.EncodeToString(addrBytes) + + account := &coreIndexerData.AlteredAccount{ + Address: addr, + Balance: "1000", + Nonce: 1, } egldAccount := &data.Account{ - UserAccount: mockAccount, + UserAccount: account, IsSender: false, } - accountsStub := &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, - } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), accountsStub, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) res := ap.PrepareRegularAccountsMap([]*data.Account{egldAccount}) - require.Equal(t, map[string]*data.AccountInfo{ - hex.EncodeToString([]byte(addr)): { - Address: hex.EncodeToString([]byte(addr)), - Nonce: 1, - Balance: "1000", - BalanceNum: balanceConverter.ComputeBalanceAsFloat(big.NewInt(1000)), - TotalBalanceWithStake: "1000", - TotalBalanceWithStakeNum: balanceConverter.ComputeBalanceAsFloat(big.NewInt(1000)), - IsSmartContract: true, - }, - }, res) + require.Equal(t, &data.AccountInfo{ + Address: addr, + Nonce: 1, + Balance: "1000", + BalanceNum: balanceConverter.ComputeBalanceAsFloat(big.NewInt(1000)), + TotalBalanceWithStake: "1000", + TotalBalanceWithStakeNum: balanceConverter.ComputeBalanceAsFloat(big.NewInt(1000)), + IsSmartContract: true, + }, + res[addr]) } func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { t.Parallel() - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - Properties: []byte("ok"), - TokenMetaData: &esdt.MetaData{ - Creator: []byte("creator"), - }, - } - addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accountsStub := &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil + + account := &coreIndexerData.AlteredAccount{ + Address: hex.EncodeToString([]byte(addr)), + Tokens: []*coreIndexerData.AccountTokenData{ + { + Balance: "1000", + Identifier: "token", + Nonce: 15, + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + }, + { + Balance: "1000", + Identifier: "token", + Nonce: 16, + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + }, }, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), accountsStub, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) accountsESDT := []*data.AccountESDT{ - {Account: mockAccount, TokenIdentifier: "token", IsNFTOperation: true, NFTNonce: 15}, - {Account: mockAccount, TokenIdentifier: "token", IsNFTOperation: true, NFTNonce: 16}, + {Account: account, TokenIdentifier: "token", IsNFTOperation: true, NFTNonce: 15}, + {Account: account, TokenIdentifier: "token", IsNFTOperation: true, NFTNonce: 16}, } res := ap.PrepareAccountsMapESDT(accountsESDT) require.Len(t, res, 2) @@ -416,7 +380,7 @@ func TestAccountsProcessor_PrepareAccountsHistory(t *testing.T) { }, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), &mock.AccountsStub{}, balanceConverter) + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) res := ap.PrepareAccountsHistory(100, accounts) accountBalanceHistory := res["addr1-token-112-10"] @@ -430,56 +394,3 @@ func TestAccountsProcessor_PrepareAccountsHistory(t *testing.T) { Identifier: "token-112-0a", }, accountBalanceHistory) } - -func TestAccountsProcessor_GetUserAccountErrors(t *testing.T) { - t.Parallel() - - localErr := errors.New("local error") - tests := []struct { - name string - argsFunc func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) - inputAddress string - exError error - }{ - { - name: "InvalidAddress", - exError: localErr, - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterStub{ - DecodeCalled: func(humanReadable string) ([]byte, error) { - return nil, localErr - }}, &mock.AccountsStub{}, balanceConverter - }, - }, - { - name: "CannotLoadAccount", - exError: localErr, - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return nil, localErr - }, - }, balanceConverter - }, - }, - { - name: "CannotCastAccount", - exError: indexer.ErrCannotCastAccountHandlerToUserAccount, - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.AccountsAdapter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return nil, nil - }, - }, balanceConverter - }, - }, - } - - for _, tt := range tests { - ap, err := NewAccountsProcessor(tt.argsFunc()) - require.Nil(t, err) - - _, err = ap.getUserAccount(tt.inputAddress) - require.Equal(t, tt.exError, err) - } -} diff --git a/process/elasticProcessor.go b/process/elasticProcessor.go index a471b481..0cd16de1 100644 --- a/process/elasticProcessor.go +++ b/process/elasticProcessor.go @@ -393,6 +393,7 @@ func (ei *elasticProcessor) SaveTransactions( body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, + coreAlteredAccounts map[string]*indexer.AlteredAccount, ) error { headerTimestamp := header.GetTimeStamp() @@ -434,7 +435,7 @@ func (ei *elasticProcessor) SaveTransactions( return err } - err = ei.indexAlteredAccounts(headerTimestamp, preparedResults.AlteredAccts, logsData.PendingBalances) + err = ei.indexAlteredAccounts(headerTimestamp, preparedResults.AlteredAccts, logsData.PendingBalances, coreAlteredAccounts) if err != nil { return err } @@ -601,8 +602,9 @@ func (ei *elasticProcessor) indexAlteredAccounts( timestamp uint64, alteredAccounts data.AlteredAccountsHandler, pendingBalances map[string]*data.AccountInfo, + coreAlteredAccounts map[string]*indexer.AlteredAccount, ) error { - regularAccountsToIndex, accountsToIndexESDT := ei.accountsProc.GetAccounts(alteredAccounts) + regularAccountsToIndex, accountsToIndexESDT := ei.accountsProc.GetAccounts(alteredAccounts, coreAlteredAccounts) err := ei.SaveAccounts(timestamp, regularAccountsToIndex) if err != nil { diff --git a/process/elasticProcessor_test.go b/process/elasticProcessor_test.go index eaa715a8..535b9010 100644 --- a/process/elasticProcessor_test.go +++ b/process/elasticProcessor_test.go @@ -48,7 +48,7 @@ func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, argume func createMockElasticProcessorArgs() *ArgElasticProcessor { balanceConverter, _ := converters.NewBalanceConverter(10) - acp, _ := accounts.NewAccountsProcessor(&mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, &mock.AccountsStub{}, balanceConverter) + acp, _ := accounts.NewAccountsProcessor(&mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, balanceConverter) bp, _ := block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) mp, _ := miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) vp, _ := validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32)) @@ -427,7 +427,7 @@ func TestElasticseachSaveTransactions(t *testing.T) { elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) pool := &indexer.Pool{Txs: txPool} - err := elasticDatabase.SaveTransactions(body, header, pool) + err := elasticDatabase.SaveTransactions(body, header, pool, nil) require.Equal(t, localErr, err) } @@ -656,7 +656,7 @@ func TestElasticProcessor_SaveTransactionsIndexSCResults(t *testing.T) { elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.enabledIndexes[elasticIndexer.ScResultsIndex] = struct{}{} - err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &indexer.Pool{}) + err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &indexer.Pool{}, nil) require.Nil(t, err) require.True(t, called) } @@ -692,7 +692,7 @@ func TestElasticProcessor_SaveTransactionsIndexReceipts(t *testing.T) { elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.enabledIndexes[elasticIndexer.ReceiptsIndex] = struct{}{} - err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &indexer.Pool{}) + err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &indexer.Pool{}, nil) require.Nil(t, err) require.True(t, called) } @@ -716,7 +716,7 @@ func TestElasticProcessor_IndexAlteredAccounts(t *testing.T) { elasticSearchProc.enabledIndexes[elasticIndexer.AccountsESDTHistoryIndex] = struct{}{} alteredAccounts := data.NewAlteredAccounts() - err := elasticSearchProc.indexAlteredAccounts(100, alteredAccounts, nil) + err := elasticSearchProc.indexAlteredAccounts(100, alteredAccounts, nil, nil) require.Nil(t, err) require.True(t, called) } diff --git a/process/factory/elasticProcessorFactory.go b/process/factory/elasticProcessorFactory.go index 7c04bdd5..8f29bd64 100644 --- a/process/factory/elasticProcessorFactory.go +++ b/process/factory/elasticProcessorFactory.go @@ -24,7 +24,6 @@ type ArgElasticProcessorFactory struct { AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter DBClient processIndexer.DatabaseClientHandler - AccountsDB indexer.AccountsAdapter ShardCoordinator indexer.ShardCoordinator TransactionFeeCalculator indexer.FeesProcessorHandler EnabledIndexes []string @@ -57,7 +56,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.Elast accountsProc, err := accounts.NewAccountsProcessor( arguments.Marshalizer, arguments.AddressPubkeyConverter, - arguments.AccountsDB, balanceConverter, ) if err != nil { diff --git a/process/factory/elasticProcessorFactory_test.go b/process/factory/elasticProcessorFactory_test.go index b7da73fe..b3aa138b 100644 --- a/process/factory/elasticProcessorFactory_test.go +++ b/process/factory/elasticProcessorFactory_test.go @@ -15,7 +15,6 @@ func TestCreateElasticProcessor(t *testing.T) { AddressPubkeyConverter: &mock.PubkeyConverterMock{}, ValidatorPubkeyConverter: &mock.PubkeyConverterMock{}, DBClient: &mock.DatabaseWriterStub{}, - AccountsDB: &mock.AccountsStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, TransactionFeeCalculator: &mock.EconomicsHandlerStub{}, EnabledIndexes: []string{"blocks"}, diff --git a/process/interface.go b/process/interface.go index 096ca1a3..b1248f1f 100644 --- a/process/interface.go +++ b/process/interface.go @@ -27,7 +27,7 @@ type DatabaseClientHandler interface { // DBAccountHandler defines the actions that an accounts handler should do type DBAccountHandler interface { - GetAccounts(alteredAccounts data.AlteredAccountsHandler) ([]*data.Account, []*data.AccountESDT) + GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*indexer.AlteredAccount) ([]*data.Account, []*data.AccountESDT) PrepareRegularAccountsMap(accounts []*data.Account) map[string]*data.AccountInfo PrepareAccountsMapESDT(accounts []*data.AccountESDT) map[string]*data.AccountInfo PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo) map[string]*data.AccountBalanceHistory diff --git a/workItems/interface.go b/workItems/interface.go index c7c98770..a9f6bb45 100644 --- a/workItems/interface.go +++ b/workItems/interface.go @@ -23,7 +23,7 @@ type saveBlockIndexer interface { txsSize int, ) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error } type saveRatingIndexer interface { diff --git a/workItems/workItemAccounts.go b/workItems/workItemAccounts.go index 0b5db49a..15f62f2c 100644 --- a/workItems/workItemAccounts.go +++ b/workItems/workItemAccounts.go @@ -2,20 +2,20 @@ package workItems import ( "github.com/ElrondNetwork/elastic-indexer-go/data" - coreData "github.com/ElrondNetwork/elrond-go-core/data" + "github.com/ElrondNetwork/elrond-go-core/data/indexer" ) type itemAccounts struct { indexer saveAccountsIndexer blockTimestamp uint64 - accounts []coreData.UserAccountHandler + accounts map[string]*indexer.AlteredAccount } // NewItemAccounts will create a new instance of itemAccounts func NewItemAccounts( indexer saveAccountsIndexer, blockTimestamp uint64, - accounts []coreData.UserAccountHandler, + accounts map[string]*indexer.AlteredAccount, ) WorkItemHandler { return &itemAccounts{ indexer: indexer, @@ -27,11 +27,13 @@ func NewItemAccounts( // Save will save information about an account func (wiv *itemAccounts) Save() error { accounts := make([]*data.Account, len(wiv.accounts)) - for idx, account := range wiv.accounts { + idx := 0 + for _, account := range wiv.accounts { accounts[idx] = &data.Account{ UserAccount: account, IsSender: false, } + idx++ } err := wiv.indexer.SaveAccounts(wiv.blockTimestamp, accounts) diff --git a/workItems/workItemAccounts_test.go b/workItems/workItemAccounts_test.go index e3af2746..341ba235 100644 --- a/workItems/workItemAccounts_test.go +++ b/workItems/workItemAccounts_test.go @@ -7,7 +7,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elastic-indexer-go/workItems" - coreData "github.com/ElrondNetwork/elrond-go-core/data" + "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/stretchr/testify/require" ) @@ -21,7 +21,7 @@ func TestItemAccounts_Save(t *testing.T) { }, }, 0, - []coreData.UserAccountHandler{}, + make(map[string]*indexer.AlteredAccount), ) require.False(t, itemAccounts.IsInterfaceNil()) @@ -39,7 +39,7 @@ func TestItemAccounts_SaveAccountsShouldErr(t *testing.T) { }, }, 0, - []coreData.UserAccountHandler{}, + make(map[string]*indexer.AlteredAccount), ) require.False(t, itemAccounts.IsInterfaceNil()) diff --git a/workItems/workItemBlock.go b/workItems/workItemBlock.go index 1e88d349..15bb306b 100644 --- a/workItems/workItemBlock.go +++ b/workItems/workItemBlock.go @@ -81,7 +81,7 @@ func (wib *itemBlock) Save() error { err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) } - err = wib.indexer.SaveTransactions(body, wib.argsSaveBlock.Header, wib.argsSaveBlock.TransactionsPool) + err = wib.indexer.SaveTransactions(body, wib.argsSaveBlock.Header, wib.argsSaveBlock.TransactionsPool, wib.argsSaveBlock.AlteredAccounts) if err != nil { return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) diff --git a/workItems/workItemBlock_test.go b/workItems/workItemBlock_test.go index f2fbf69a..812851e2 100644 --- a/workItems/workItemBlock_test.go +++ b/workItems/workItemBlock_test.go @@ -83,7 +83,7 @@ func TestItemBlock_SaveNoMiniblocksShoulCallSaveHeader(t *testing.T) { countCalled++ return nil }, - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool) error { + SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { countCalled++ return nil }, @@ -127,7 +127,7 @@ func TestItemBlock_SaveTransactionsShouldErr(t *testing.T) { localErr := errors.New("local err") itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool) error { + SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { return localErr }, }, @@ -156,7 +156,7 @@ func TestItemBlock_SaveShouldWork(t *testing.T) { countCalled++ return nil }, - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool) error { + SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { countCalled++ return nil }, From fafd4b190155af66f3ea33f8cd6632451a26db2d Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Tue, 1 Feb 2022 14:36:16 +0200 Subject: [PATCH 02/96] update core dep --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4375b9e3..3a227d20 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.15 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.8-0.20220128143134-2fee54561061 + github.com/ElrondNetwork/elrond-go-core v1.1.10-0.20220201122512-86d7af4818f4 github.com/ElrondNetwork/elrond-go-logger v1.0.5 github.com/ElrondNetwork/elrond-vm-common v1.2.9 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 09c42a48..2ec5f034 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.8-0.20220128143134-2fee54561061 h1:+qjz6/oMTVU2sp2MCUYpTvxpT63Dgm8jHDboqkEhTGQ= -github.com/ElrondNetwork/elrond-go-core v1.1.8-0.20220128143134-2fee54561061/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= +github.com/ElrondNetwork/elrond-go-core v1.1.10-0.20220201122512-86d7af4818f4 h1:XosXNM9wyX12VbhV335Tzswrcj8Iq3f+1lLf0slttPo= +github.com/ElrondNetwork/elrond-go-core v1.1.10-0.20220201122512-86d7af4818f4/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.5 h1:tB/HBvV9IVeCaSrGakX+GLGu7K5UPLv8gA0TNKPOTOU= github.com/ElrondNetwork/elrond-go-logger v1.0.5/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= @@ -40,6 +40,7 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= From 82657e8d9d94a48d5b953a25a04e39b89e3b6404 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Mon, 7 Feb 2022 16:10:57 +0200 Subject: [PATCH 03/96] fixes after merge master --- go.mod | 2 +- go.sum | 2 ++ process/accounts/accountsProcessor.go | 39 +++++---------------------- 3 files changed, 10 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 5c1ee6fb..ea5421ef 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.15 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.9 + github.com/ElrondNetwork/elrond-go-core v1.1.12-0.20220207134829-074fcd3468c6 github.com/ElrondNetwork/elrond-go-logger v1.0.5 github.com/ElrondNetwork/elrond-vm-common v1.2.9 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index d79e2c6a..1608d722 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6y github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-core v1.1.9 h1:ds+63MZp+bGKXmOCiRNVssENf60TGe62K7TOpSP1WUg= github.com/ElrondNetwork/elrond-go-core v1.1.9/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.12-0.20220207134829-074fcd3468c6 h1:FkIZFQxAMyL7khQuTIWDHli/F3CplpQ/tjHfijvmnFI= +github.com/ElrondNetwork/elrond-go-core v1.1.12-0.20220207134829-074fcd3468c6/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.5 h1:tB/HBvV9IVeCaSrGakX+GLGu7K5UPLv8gA0TNKPOTOU= github.com/ElrondNetwork/elrond-go-logger v1.0.5/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index f921da49..8e6bfa8f 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -15,7 +15,6 @@ import ( coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" ) var log = logger.GetOrCreate("indexer/process/accounts") @@ -229,40 +228,16 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In } if esdtToken.TokenMetaData == nil && accountESDT.NFTNonce > 0 { - metadata, errLoad := ap.loadMetadataFromSystemAccount(tokenKey) - if errLoad != nil { - return nil, "", nil, errLoad - } - - esdtToken.TokenMetaData = metadata + // TODO: refactor when elrond-go treats NFTs from sys account and saves it in altered accounts + //metadata, errLoad := ap.loadMetadataFromSystemAccount(tokenKey) + //if errLoad != nil { + // return nil, "", nil, errLoad + //} + // + //esdtToken.TokenMetaData = metadata } tokenMetaData := converters.PrepareTokenMetaData(ap.addressPubkeyConverter, esdtToken) return esdtToken.Value, hex.EncodeToString(esdtToken.Properties), tokenMetaData, nil } - -func (ap *accountsProcessor) loadMetadataFromSystemAccount(tokenKey []byte) (*esdt.MetaData, error) { - systemAccount, err := ap.accountsDB.LoadAccount(vmcommon.SystemAccountAddress) - if err != nil { - return nil, err - } - - userAccount, ok := systemAccount.(coreData.UserAccountHandler) - if !ok { - return nil, indexer.ErrCannotCastAccountHandlerToUserAccount - } - - marshaledData, err := userAccount.RetrieveValueFromDataTrieTracker(tokenKey) - if err != nil { - return nil, err - } - - esdtData := &esdt.ESDigitalToken{} - err = ap.internalMarshalizer.Unmarshal(esdtData, marshaledData) - if err != nil { - return nil, err - } - - return esdtData.TokenMetaData, nil -} From ffccec7cd962562ce02d98c8c25c4dc2653355df Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 9 Feb 2022 17:12:53 +0200 Subject: [PATCH 04/96] new core reference --- go.mod | 2 +- go.sum | 6 ++---- process/accounts/accountsProcessor.go | 10 ---------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index ea5421ef..87c99605 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.15 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.12-0.20220207134829-074fcd3468c6 + github.com/ElrondNetwork/elrond-go-core v1.1.13 github.com/ElrondNetwork/elrond-go-logger v1.0.5 github.com/ElrondNetwork/elrond-vm-common v1.2.9 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 1608d722..fb1bec64 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.9 h1:ds+63MZp+bGKXmOCiRNVssENf60TGe62K7TOpSP1WUg= -github.com/ElrondNetwork/elrond-go-core v1.1.9/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= -github.com/ElrondNetwork/elrond-go-core v1.1.12-0.20220207134829-074fcd3468c6 h1:FkIZFQxAMyL7khQuTIWDHli/F3CplpQ/tjHfijvmnFI= -github.com/ElrondNetwork/elrond-go-core v1.1.12-0.20220207134829-074fcd3468c6/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.13 h1:67o8RRagno509HWJwnyXTShOnLY6PAP3n5ok/B8sgQY= +github.com/ElrondNetwork/elrond-go-core v1.1.13/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.5 h1:tB/HBvV9IVeCaSrGakX+GLGu7K5UPLv8gA0TNKPOTOU= github.com/ElrondNetwork/elrond-go-logger v1.0.5/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index 8e6bfa8f..c0183500 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -227,16 +227,6 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In return big.NewInt(0), "", nil, nil } - if esdtToken.TokenMetaData == nil && accountESDT.NFTNonce > 0 { - // TODO: refactor when elrond-go treats NFTs from sys account and saves it in altered accounts - //metadata, errLoad := ap.loadMetadataFromSystemAccount(tokenKey) - //if errLoad != nil { - // return nil, "", nil, errLoad - //} - // - //esdtToken.TokenMetaData = metadata - } - tokenMetaData := converters.PrepareTokenMetaData(ap.addressPubkeyConverter, esdtToken) return esdtToken.Value, hex.EncodeToString(esdtToken.Properties), tokenMetaData, nil From 2a53c6bc2151f8eaa1b3bca85b436178faf4de94 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 9 Feb 2022 18:53:33 +0200 Subject: [PATCH 05/96] fixes after review --- interface.go | 6 ------ mock/accountsStub.go | 23 ----------------------- process/accounts/accountsProcessor.go | 11 +++++++++-- workItems/workItemAccounts.go | 6 ++---- 4 files changed, 11 insertions(+), 35 deletions(-) delete mode 100644 mock/accountsStub.go diff --git a/interface.go b/interface.go index 3d74c415..1351dbde 100644 --- a/interface.go +++ b/interface.go @@ -8,7 +8,6 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/indexer" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" ) // DispatcherHandler defines the interface for the dispatcher that will manage when items are saved in elasticsearch database @@ -71,11 +70,6 @@ type Indexer interface { IsNilIndexer() bool } -type AccountsAdapter interface { - LoadAccount(address []byte) (vmcommon.AccountHandler, error) - IsInterfaceNil() bool -} - // BalanceConverter defines what a balance converter should be able to do type BalanceConverter interface { ComputeBalanceAsFloat(balance *big.Int) float64 diff --git a/mock/accountsStub.go b/mock/accountsStub.go deleted file mode 100644 index a241362d..00000000 --- a/mock/accountsStub.go +++ /dev/null @@ -1,23 +0,0 @@ -package mock - -import ( - vmcommon "github.com/ElrondNetwork/elrond-vm-common" -) - -// AccountsStub - -type AccountsStub struct { - LoadAccountCalled func(container []byte) (vmcommon.AccountHandler, error) -} - -// LoadAccount - -func (as *AccountsStub) LoadAccount(address []byte) (vmcommon.AccountHandler, error) { - if as.LoadAccountCalled != nil { - return as.LoadAccountCalled(address) - } - return nil, nil -} - -// IsInterfaceNil returns true if there is no value under the interface -func (as *AccountsStub) IsInterfaceNil() bool { - return as == nil -} diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index c0183500..fb9c9599 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -62,6 +62,7 @@ func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHan for address, altered := range allAlteredAccounts { alteredAccount := coreAlteredAccounts[address] if alteredAccount == nil { + log.Warn("account not found in core altered accounts map", "address", address) continue } @@ -76,7 +77,8 @@ func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHan func splitAlteredAccounts( account *coreIndexerData.AlteredAccount, - altered []*data.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { + altered []*data.AlteredAccount, +) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) for _, info := range altered { @@ -120,7 +122,12 @@ func (ap *accountsProcessor) PrepareRegularAccountsMap(accounts []*data.Account) log.Warn("PrepareRegularAccountsMap: cannot decode address", "address", address, "error", err) continue } - balance, _ := big.NewInt(0).SetString(userAccount.UserAccount.Balance, 10) + balance, ok := big.NewInt(0).SetString(userAccount.UserAccount.Balance, 10) + if !ok { + log.Warn("cannot cast account's balance to big int", "value", userAccount.UserAccount.Balance) + continue + } + balanceAsFloat := ap.balanceConverter.ComputeBalanceAsFloat(balance) acc := &data.AccountInfo{ Address: address, diff --git a/workItems/workItemAccounts.go b/workItems/workItemAccounts.go index 15f62f2c..d3289ca7 100644 --- a/workItems/workItemAccounts.go +++ b/workItems/workItemAccounts.go @@ -27,13 +27,11 @@ func NewItemAccounts( // Save will save information about an account func (wiv *itemAccounts) Save() error { accounts := make([]*data.Account, len(wiv.accounts)) - idx := 0 for _, account := range wiv.accounts { - accounts[idx] = &data.Account{ + accounts = append(accounts, &data.Account{ UserAccount: account, IsSender: false, - } - idx++ + }) } err := wiv.indexer.SaveAccounts(wiv.blockTimestamp, accounts) From 87a6eaa5c8d438dfbe65624cc1ef20859e9b3302 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 9 Feb 2022 19:23:32 +0200 Subject: [PATCH 06/96] small fix --- workItems/workItemAccounts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workItems/workItemAccounts.go b/workItems/workItemAccounts.go index d3289ca7..aa7068c3 100644 --- a/workItems/workItemAccounts.go +++ b/workItems/workItemAccounts.go @@ -26,7 +26,7 @@ func NewItemAccounts( // Save will save information about an account func (wiv *itemAccounts) Save() error { - accounts := make([]*data.Account, len(wiv.accounts)) + accounts := make([]*data.Account, 0, len(wiv.accounts)) for _, account := range wiv.accounts { accounts = append(accounts, &data.Account{ UserAccount: account, From ab71fb4606d19990a741b43d105c223229650d45 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Fri, 18 Mar 2022 15:20:39 +0200 Subject: [PATCH 07/96] fixes after merge --- go.sum | 2 -- mock/dbAccountsHandlerStub.go | 2 +- process/accounts/accountsProcessor.go | 44 ++++++--------------------- process/accounts/errors.go | 5 +++ process/elasticProcessor.go | 6 ++-- process/interface.go | 2 +- 6 files changed, 20 insertions(+), 41 deletions(-) create mode 100644 process/accounts/errors.go diff --git a/go.sum b/go.sum index eaae992c..c21f6c5f 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.9 h1:ds+63MZp+bGKXmOCiRNVssENf60TGe62K7TOpSP1WUg= -github.com/ElrondNetwork/elrond-go-core v1.1.9/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-core v1.1.14 h1:JKpeI+1US4FuE8NwN3dqe0HUTYKLQuYKvwbTqhGt334= github.com/ElrondNetwork/elrond-go-core v1.1.14/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= diff --git a/mock/dbAccountsHandlerStub.go b/mock/dbAccountsHandlerStub.go index 9f8bd6b4..a124a5d2 100644 --- a/mock/dbAccountsHandlerStub.go +++ b/mock/dbAccountsHandlerStub.go @@ -56,5 +56,5 @@ func (dba *DBAccountsHandlerStub) SerializeNFTCreateInfo(_ []*data.TokenInfo) ([ } // PutTokenMedataDataInTokens - -func (dba *DBAccountsHandlerStub) PutTokenMedataDataInTokens(_ []*data.TokenInfo) { +func (dba *DBAccountsHandlerStub) PutTokenMedataDataInTokens(_ []*data.TokenInfo, _ map[string]*indexer.AlteredAccount) { } diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index f87ffb55..e732bc38 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -240,14 +240,13 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In } // PutTokenMedataDataInTokens will put the TokenMedata in provided tokens data -func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.TokenInfo) { +func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount) { for _, tokenData := range tokensData { if tokenData.Data != nil || tokenData.Nonce == 0 { continue } - tokenKey := computeTokenKey(tokenData.Token, tokenData.Nonce) - metadata, errLoad := ap.loadMetadataFromSystemAccount(tokenKey) + metadata, errLoad := ap.loadMetadataForToken(tokenData, coreAlteredAccounts) if errLoad != nil { log.Warn("cannot load token metadata", "token identifier ", tokenData.Identifier, @@ -260,37 +259,14 @@ func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.Token } } -func (ap *accountsProcessor) loadMetadataFromSystemAccount(tokenKey []byte) (*esdt.MetaData, error) { - systemAccount, err := ap.accountsDB.LoadAccount(vmcommon.SystemAccountAddress) - if err != nil { - return nil, err - } - - userAccount, ok := systemAccount.(coreData.UserAccountHandler) - if !ok { - return nil, indexer.ErrCannotCastAccountHandlerToUserAccount - } - - marshaledData, err := userAccount.RetrieveValueFromDataTrieTracker(tokenKey) - if err != nil { - return nil, err - } - - esdtData := &esdt.ESDigitalToken{} - err = ap.internalMarshalizer.Unmarshal(esdtData, marshaledData) - if err != nil { - return nil, err - } - - return esdtData.TokenMetaData, nil -} - -func computeTokenKey(token string, nonce uint64) []byte { - tokenKey := []byte(core.ElrondProtectedKeyPrefix + core.ESDTKeyIdentifier + token) - if nonce > 0 { - nonceBig := big.NewInt(0).SetUint64(nonce) - tokenKey = append(tokenKey, nonceBig.Bytes()...) +func (ap *accountsProcessor) loadMetadataForToken(tokenData *data.TokenInfo, coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount) (*esdt.MetaData, error) { + for _, account := range coreAlteredAccounts { + for _, token := range account.Tokens { + if tokenData.Token == token.Identifier && tokenData.Nonce == token.Nonce { + return token.MetaData, nil + } + } } - return tokenKey + return nil, fmt.Errorf("%w for identifier %s and nonce %d", errTokenNotFound, tokenData.Identifier, tokenData.Nonce) } diff --git a/process/accounts/errors.go b/process/accounts/errors.go new file mode 100644 index 00000000..16ba4517 --- /dev/null +++ b/process/accounts/errors.go @@ -0,0 +1,5 @@ +package accounts + +import "errors" + +var errTokenNotFound = errors.New("token not found") diff --git a/process/elasticProcessor.go b/process/elasticProcessor.go index 0c96e67b..c6217742 100644 --- a/process/elasticProcessor.go +++ b/process/elasticProcessor.go @@ -415,7 +415,7 @@ func (ei *elasticProcessor) SaveTransactions( return err } - err = ei.indexNFTCreateInfo(logsData.Tokens) + err = ei.indexNFTCreateInfo(logsData.Tokens, coreAlteredAccounts) if err != nil { return err } @@ -652,7 +652,7 @@ func (ei *elasticProcessor) indexAccountsESDT(accountsESDTMap map[string]*data.A return ei.serializeAndIndexAccounts(accountsESDTMap, elasticIndexer.AccountsESDTIndex, true) } -func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler) error { +func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { shouldSkipIndex := !ei.isIndexEnabled(elasticIndexer.TokensIndex) || tokensData.Len() == 0 if shouldSkipIndex { return nil @@ -667,7 +667,7 @@ func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler) er tokensData.AddTypeFromResponse(responseTokens) tokens := tokensData.GetAll() - ei.accountsProc.PutTokenMedataDataInTokens(tokens) + ei.accountsProc.PutTokenMedataDataInTokens(tokens, coreAlteredAccounts) buffSlice, err := ei.accountsProc.SerializeNFTCreateInfo(tokens) if err != nil { diff --git a/process/interface.go b/process/interface.go index fc6da6fb..5102ef76 100644 --- a/process/interface.go +++ b/process/interface.go @@ -31,7 +31,7 @@ type DBAccountHandler interface { PrepareRegularAccountsMap(accounts []*data.Account) map[string]*data.AccountInfo PrepareAccountsMapESDT(accounts []*data.AccountESDT) map[string]*data.AccountInfo PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo) map[string]*data.AccountBalanceHistory - PutTokenMedataDataInTokens(tokensData []*data.TokenInfo) + PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*indexer.AlteredAccount) SerializeAccountsHistory(accounts map[string]*data.AccountBalanceHistory) ([]*bytes.Buffer, error) SerializeAccounts(accounts map[string]*data.AccountInfo, areESDTAccounts bool) ([]*bytes.Buffer, error) From 251f84d51fbfa710c61b75e25b15f615a18d9eb8 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Tue, 3 May 2022 11:00:16 +0300 Subject: [PATCH 08/96] added unit tests --- process/accounts/accountsProcessor_test.go | 81 ++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/process/accounts/accountsProcessor_test.go b/process/accounts/accountsProcessor_test.go index 59c0a18a..9b4ba2c5 100644 --- a/process/accounts/accountsProcessor_test.go +++ b/process/accounts/accountsProcessor_test.go @@ -394,3 +394,84 @@ func TestAccountsProcessor_PrepareAccountsHistory(t *testing.T) { Identifier: "token-112-0a", }, accountBalanceHistory) } + +func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { + t.Parallel() + + t.Run("no tokens with missing data or nonce higher than 0", func(t *testing.T) { + t.Parallel() + + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + + oldCreator := "old creator" + tokensInfo := []*data.TokenInfo{ + {Data: nil}, {Nonce: 5, Data: &data.TokenMetaData{Creator: oldCreator}}, + } + emptyAlteredAccounts := map[string]*coreIndexerData.AlteredAccount{} + ap.PutTokenMedataDataInTokens(tokensInfo, emptyAlteredAccounts) + require.Empty(t, emptyAlteredAccounts) + require.Empty(t, tokensInfo[0].Data) + require.Equal(t, oldCreator, tokensInfo[1].Data.Creator) + }) + + t.Run("error loading token, should not update metadata", func(t *testing.T) { + t.Parallel() + + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + + tokensInfo := []*data.TokenInfo{ + { + Name: "token0", + Data: nil, + Nonce: 5, + }, + } + + alteredAccounts := map[string]*coreIndexerData.AlteredAccount{ + "addr": {Tokens: []*coreIndexerData.AccountTokenData{}}, + } + ap.PutTokenMedataDataInTokens(tokensInfo, alteredAccounts) + require.Empty(t, tokensInfo[0].Data) + }) + + t.Run("should work and update metadata", func(t *testing.T) { + t.Parallel() + + ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + + metadata0, metadata1 := &esdt.MetaData{Creator: []byte("creator 0")}, &esdt.MetaData{Creator: []byte("creator 1")} + tokensInfo := []*data.TokenInfo{ + { + Nonce: 5, + Token: "token0-5t6y7u", + Identifier: "token0-5t6y7u-05", + }, + { + Nonce: 10, + Token: "token1-999ddd", + Identifier: "token1-999ddd-0a", + }, + } + + alteredAccounts := map[string]*coreIndexerData.AlteredAccount{ + "addr0": { + Tokens: []*coreIndexerData.AccountTokenData{ + { + Identifier: "token0-5t6y7u", + Nonce: 5, + MetaData: metadata0, + }, + { + Identifier: "token1-999ddd", + Nonce: 10, + MetaData: metadata1, + }, + }, + }, + } + + ap.PutTokenMedataDataInTokens(tokensInfo, alteredAccounts) + require.Equal(t, hex.EncodeToString(metadata0.Creator), tokensInfo[0].Data.Creator) + require.Equal(t, hex.EncodeToString(metadata1.Creator), tokensInfo[1].Data.Creator) + }) +} From f2821613bbe7f05435f1dc0cc0e2f0c4d2818788 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Tue, 3 May 2022 12:11:54 +0300 Subject: [PATCH 09/96] fixes after review --- process/accounts/accountsProcessor.go | 22 ++++------ process/accounts/accountsProcessor_test.go | 50 +++++++++------------- process/elasticProcessor_test.go | 2 +- process/factory/elasticProcessorFactory.go | 1 - 4 files changed, 31 insertions(+), 44 deletions(-) diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index e732bc38..6af9493c 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -13,7 +13,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data/esdt" coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" - "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" ) @@ -21,20 +20,15 @@ var log = logger.GetOrCreate("indexer/process/accounts") // accountsProcessor a is structure responsible for processing accounts type accountsProcessor struct { - internalMarshalizer marshal.Marshalizer addressPubkeyConverter core.PubkeyConverter balanceConverter indexer.BalanceConverter } // NewAccountsProcessor will create a new instance of accounts processor func NewAccountsProcessor( - marshalizer marshal.Marshalizer, addressPubkeyConverter core.PubkeyConverter, balanceConverter indexer.BalanceConverter, ) (*accountsProcessor, error) { - if check.IfNil(marshalizer) { - return nil, indexer.ErrNilMarshalizer - } if check.IfNil(addressPubkeyConverter) { return nil, indexer.ErrNilPubkeyConverter } @@ -43,7 +37,6 @@ func NewAccountsProcessor( } return &accountsProcessor{ - internalMarshalizer: marshalizer, addressPubkeyConverter: addressPubkeyConverter, balanceConverter: balanceConverter, }, nil @@ -119,12 +112,12 @@ func (ap *accountsProcessor) PrepareRegularAccountsMap(accounts []*data.Account) address := userAccount.UserAccount.Address addressBytes, err := ap.addressPubkeyConverter.Decode(address) if err != nil { - log.Warn("PrepareRegularAccountsMap: cannot decode address", "address", address, "error", err) + log.Warn("accountsProcessor.PrepareRegularAccountsMap: cannot decode address", "address", address, "error", err) continue } balance, ok := big.NewInt(0).SetString(userAccount.UserAccount.Balance, 10) if !ok { - log.Warn("cannot cast account's balance to big int", "value", userAccount.UserAccount.Balance) + log.Warn("accountsProcessor.PrepareRegularAccountsMap: cannot cast account's balance to big int", "value", userAccount.UserAccount.Balance) continue } @@ -155,12 +148,12 @@ func (ap *accountsProcessor) PrepareAccountsMapESDT( address := accountESDT.Account.Address addressBytes, err := ap.addressPubkeyConverter.Decode(address) if err != nil { - log.Warn("PrepareAccountsMapESDT: cannot decode address", "address", address, "error", err) + log.Warn("accountsProcessor.PrepareAccountsMapESDT: cannot decode address", "address", address, "error", err) continue } balance, properties, tokenMetaData, err := ap.getESDTInfo(accountESDT) if err != nil { - log.Warn("cannot get esdt info from account", + log.Warn("accountsProcessor.PrepareAccountsMapESDT: cannot get esdt info from account", "address", address, "error", err.Error()) continue @@ -248,7 +241,7 @@ func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.Token metadata, errLoad := ap.loadMetadataForToken(tokenData, coreAlteredAccounts) if errLoad != nil { - log.Warn("cannot load token metadata", + log.Warn("accountsProcessor.PutTokenMedataDataInTokens: cannot load token metadata", "token identifier ", tokenData.Identifier, "error", errLoad.Error()) @@ -259,7 +252,10 @@ func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.Token } } -func (ap *accountsProcessor) loadMetadataForToken(tokenData *data.TokenInfo, coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount) (*esdt.MetaData, error) { +func (ap *accountsProcessor) loadMetadataForToken( + tokenData *data.TokenInfo, + coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount, +) (*esdt.MetaData, error) { for _, account := range coreAlteredAccounts { for _, token := range account.Tokens { if tokenData.Token == token.Identifier && tokenData.Nonce == token.Nonce { diff --git a/process/accounts/accountsProcessor_test.go b/process/accounts/accountsProcessor_test.go index 9b4ba2c5..bfbfb9fd 100644 --- a/process/accounts/accountsProcessor_test.go +++ b/process/accounts/accountsProcessor_test.go @@ -14,7 +14,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/esdt" coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" - "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/stretchr/testify/require" ) @@ -25,34 +24,27 @@ func TestNewAccountsProcessor(t *testing.T) { tests := []struct { name string - argsFunc func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) + argsFunc func() (core.PubkeyConverter, indexer.BalanceConverter) exError error }{ { name: "NilBalanceConverter", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, nil + argsFunc: func() (core.PubkeyConverter, indexer.BalanceConverter) { + return &mock.PubkeyConverterMock{}, nil }, exError: indexer.ErrNilBalanceConverter, }, - { - name: "NilMarshalizer", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { - return nil, &mock.PubkeyConverterMock{}, balanceConverter - }, - exError: indexer.ErrNilMarshalizer, - }, { name: "NilPubKeyConverter", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, nil, balanceConverter + argsFunc: func() (core.PubkeyConverter, indexer.BalanceConverter) { + return nil, balanceConverter }, exError: indexer.ErrNilPubkeyConverter, }, { name: "ShouldWork", - argsFunc: func() (marshal.Marshalizer, core.PubkeyConverter, indexer.BalanceConverter) { - return &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, balanceConverter + argsFunc: func() (core.PubkeyConverter, indexer.BalanceConverter) { + return &mock.PubkeyConverterMock{}, balanceConverter }, exError: nil, }, @@ -69,7 +61,7 @@ func TestNewAccountsProcessor(t *testing.T) { func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) regularAccounts, esdtAccounts := ap.GetAccounts(nil, nil) require.Len(t, regularAccounts, 0) @@ -79,7 +71,7 @@ func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { func TestAccountsProcessor_PrepareRegularAccountsMapWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) accountsInfo := ap.PrepareRegularAccountsMap(nil) require.Len(t, accountsInfo, 0) @@ -88,7 +80,7 @@ func TestAccountsProcessor_PrepareRegularAccountsMapWithNil(t *testing.T) { func TestGetESDTInfo(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) tokenIdentifier := "token-001" @@ -116,7 +108,7 @@ func TestGetESDTInfo(t *testing.T) { func TestGetESDTInfoNFT(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) tokenIdentifier := "token-001" @@ -148,7 +140,7 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { t.Parallel() pubKeyConverter := mock.NewPubkeyConverterMock(32) - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, pubKeyConverter, balanceConverter) + ap, _ := NewAccountsProcessor(pubKeyConverter, balanceConverter) require.NotNil(t, ap) nftName := "Test-nft" @@ -198,7 +190,7 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -227,7 +219,7 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -250,7 +242,7 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -286,7 +278,7 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { IsSender: false, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) res := ap.PrepareRegularAccountsMap([]*data.Account{egldAccount}) @@ -330,7 +322,7 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { }, }, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) accountsESDT := []*data.AccountESDT{ @@ -380,7 +372,7 @@ func TestAccountsProcessor_PrepareAccountsHistory(t *testing.T) { }, } - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) res := ap.PrepareAccountsHistory(100, accounts) accountBalanceHistory := res["addr1-token-112-10"] @@ -401,7 +393,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("no tokens with missing data or nonce higher than 0", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) oldCreator := "old creator" tokensInfo := []*data.TokenInfo{ @@ -417,7 +409,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("error loading token, should not update metadata", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) tokensInfo := []*data.TokenInfo{ { @@ -437,7 +429,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("should work and update metadata", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(&mock.MarshalizerMock{}, mock.NewPubkeyConverterMock(32), balanceConverter) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) metadata0, metadata1 := &esdt.MetaData{Creator: []byte("creator 0")}, &esdt.MetaData{Creator: []byte("creator 1")} tokensInfo := []*data.TokenInfo{ diff --git a/process/elasticProcessor_test.go b/process/elasticProcessor_test.go index 535b9010..9c8876ba 100644 --- a/process/elasticProcessor_test.go +++ b/process/elasticProcessor_test.go @@ -48,7 +48,7 @@ func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, argume func createMockElasticProcessorArgs() *ArgElasticProcessor { balanceConverter, _ := converters.NewBalanceConverter(10) - acp, _ := accounts.NewAccountsProcessor(&mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}, balanceConverter) + acp, _ := accounts.NewAccountsProcessor(&mock.PubkeyConverterMock{}, balanceConverter) bp, _ := block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) mp, _ := miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) vp, _ := validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32)) diff --git a/process/factory/elasticProcessorFactory.go b/process/factory/elasticProcessorFactory.go index 8f29bd64..0c4d2fa0 100644 --- a/process/factory/elasticProcessorFactory.go +++ b/process/factory/elasticProcessorFactory.go @@ -54,7 +54,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.Elast } accountsProc, err := accounts.NewAccountsProcessor( - arguments.Marshalizer, arguments.AddressPubkeyConverter, balanceConverter, ) From dbbe35b68318b229f9d9e3d84eb8d64cc3783f68 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 18 May 2022 15:15:21 +0300 Subject: [PATCH 10/96] fixes after merge --- process/elasticProcessor.go | 4 ++-- process/elasticProcessor_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/process/elasticProcessor.go b/process/elasticProcessor.go index dea2c0a1..b882f49b 100644 --- a/process/elasticProcessor.go +++ b/process/elasticProcessor.go @@ -391,7 +391,7 @@ func (ei *elasticProcessor) SaveTransactions( return err } - err = ei.indexAlteredAccounts(headerTimestamp, preparedResults.AlteredAccts, logsData.PendingBalances, coreAlteredAccounts, buffers) + err = ei.indexAlteredAccounts(headerTimestamp, preparedResults.AlteredAccts, logsData.NFTsDataUpdates, coreAlteredAccounts, buffers) if err != nil { return err } @@ -570,8 +570,8 @@ func (ei *elasticProcessor) indexAlteredAccounts( timestamp uint64, alteredAccounts data.AlteredAccountsHandler, updatesNFTsData []*data.NFTDataUpdate, - buffSlice *data.BufferSlice, coreAlteredAccounts map[string]*indexer.AlteredAccount, + buffSlice *data.BufferSlice, ) error { regularAccountsToIndex, accountsToIndexESDT := ei.accountsProc.GetAccounts(alteredAccounts, coreAlteredAccounts) diff --git a/process/elasticProcessor_test.go b/process/elasticProcessor_test.go index d88e45f4..0a82bccf 100644 --- a/process/elasticProcessor_test.go +++ b/process/elasticProcessor_test.go @@ -684,7 +684,7 @@ func TestElasticProcessor_IndexAlteredAccounts(t *testing.T) { buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) alteredAccounts := data.NewAlteredAccounts() - err := elasticSearchProc.indexAlteredAccounts(100, alteredAccounts, nil, buffSlice) + err := elasticSearchProc.indexAlteredAccounts(100, alteredAccounts, nil, nil, buffSlice) require.Nil(t, err) require.True(t, called) } From b4cbee684c0d36d772db243ca528e813b8741671 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 18 May 2022 15:22:03 +0300 Subject: [PATCH 11/96] adapt integration tests --- .../accountsBalanceWithLowerTimestamp_test.go | 72 +++++------ .../accountsESDTWithTokenType_test.go | 112 +++++++++--------- 2 files changed, 88 insertions(+), 96 deletions(-) diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 98bcc722..5efc3805 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -3,7 +3,7 @@ package integrationtests import ( - "encoding/json" + "encoding/hex" "math/big" "testing" @@ -15,7 +15,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) @@ -25,7 +24,6 @@ func TestIndexAccountsBalance(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} // ################ UPDATE ACCOUNT-ESDT BALANCE ########################## @@ -40,20 +38,28 @@ func TestIndexAccountsBalance(t *testing.T) { } addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) + addr2 := "eeeebbbb" + encodedAddr := hex.EncodeToString([]byte(addr)) + encodedAddr2 := hex.EncodeToString([]byte(addr2)) + + alteredAccount := &indexer.AlteredAccount{ + Address: encodedAddr, + Balance: "0", + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "TTTT-abcd", + Balance: "1000", + Nonce: 0, + }, }, } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, + + coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + encodedAddr: alteredAccount, + encodedAddr2: alteredAccount, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header := &dataBlock.Header{ @@ -79,7 +85,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids := []string{"6161616162626262"} @@ -100,11 +106,8 @@ func TestIndexAccountsBalance(t *testing.T) { Round: 51, TimeStamp: 5000, } - mockAccount.GetBalanceCalled = func() *big.Int { - return big.NewInt(1000) - } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -124,9 +127,8 @@ func TestIndexAccountsBalance(t *testing.T) { Round: 51, TimeStamp: 6000, } - mockAccount.GetBalanceCalled = func() *big.Int { - return big.NewInt(2000) - } + + coreAlteredAccounts[encodedAddr].Balance = "2000" pool = &indexer.Pool{ Txs: map[string]coreData.TransactionHandler{ @@ -159,7 +161,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -177,32 +179,20 @@ func TestIndexAccountsBalance(t *testing.T) { //////////////////////// DELETE ESDT BALANCE LOWER TIMESTAMP //////////////// esdtToken.Value = big.NewInt(0) - mockAccount = &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, - } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + encodedAddr = hex.EncodeToString([]byte(addr)) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header = &dataBlock.Header{ Round: 51, TimeStamp: 6001, } - mockAccount.GetBalanceCalled = func() *big.Int { - return big.NewInt(2000) - } + + coreAlteredAccounts[encodedAddr].Balance = "2000" + coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" pool.Txs = make(map[string]coreData.TransactionHandler) - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = []string{"6161616162626262"} diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 9cc6c24d..deec35fc 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "math/big" "testing" @@ -16,17 +17,25 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) +const ( + expectedTokenAfterIssue = `{"name":"SEMI-token","ticker":"SEM","token":"SEMI-abcd","issuer":"61646472","currentOwner":"61646472","type":"SemiFungibleESDT","timestamp":5040,"ownersHistory":[{"address":"61646472","timestamp":5040}]}` + expectedAccountESDT = `{"address":"6161616162626262","balance":"1000","balanceNum":1e-15,"token":"SEMI-abcd","identifier":"SEMI-abcd-02","tokenNonce":2,"properties":"6f6b","data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"timestamp":5600,"type":"SemiFungibleESDT"}` + + expectedAccountsESDTWithoutType = `{"address":"6161616162626262","balance":"1000","balanceNum":1e-15,"token":"TTTT-abcd","identifier":"TTTT-abcd-02","tokenNonce":2,"properties":"6f6b","data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"timestamp":5600}` + expectedSemiFungibleToken = `{"name":"TTTT-token","ticker":"SEM","token":"TTTT-abcd","issuer":"61646472","currentOwner":"61646472","type":"SemiFungibleESDT","timestamp":5040,"ownersHistory":[{"address":"61646472","timestamp":5040}]}` + expectedAccountsESDTWithType = `{"address":"6161616162626262","balance":"1000","balanceNum":1.0E-15,"token":"TTTT-abcd","identifier":"TTTT-abcd-02","tokenNonce":2,"properties":"6f6b","data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"timestamp":5600,"type":"SemiFungibleESDT"}` + expectedSemiFungibleTokenAfterCreate = `{"identifier":"TTTT-abcd-02","token":"TTTT-abcd","nonce":2,"timestamp":5600,"data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"type":"SemiFungibleESDT"}` +) + func TestIndexAccountESDTWithTokenType(t *testing.T) { setLogLevelDebug() esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} // ################ ISSUE NON FUNGIBLE TOKEN ########################## @@ -34,7 +43,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) body := &dataBlock.Body{} @@ -61,43 +70,40 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids := []string{"SEMI-abcd"} genericResponse := &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/token-after-issue.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, expectedTokenAfterIssue, string(genericResponse.Docs[0].Source)) // ################ CREATE SEMI FUNGIBLE TOKEN ########################## shardCoordinator = &mock.ShardCoordinatorMock{ SelfID: 0, } - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - Properties: []byte("ok"), - TokenMetaData: &esdt.MetaData{ - Creator: []byte("creator"), - }, - } - addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil + encodedAddr := hex.EncodeToString([]byte(addr)) + coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + encodedAddr: { + Address: encodedAddr, + Balance: "1000", + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "SEMI-abcd", + Balance: "1000", + Nonce: 2, + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + }, + }, }, } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header = &dataBlock.Header{ @@ -130,14 +136,14 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = []string{"6161616162626262-SEMI-abcd-02"} genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, expectedAccountESDT, string(genericResponse.Docs[0].Source)) } @@ -147,7 +153,6 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} // ################ CREATE SEMI FUNGIBLE TOKEN ########################## @@ -157,29 +162,26 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) body := &dataBlock.Body{} - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - Properties: []byte("ok"), - TokenMetaData: &esdt.MetaData{ - Creator: []byte("creator"), - }, - } - addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil + encodedAddr := hex.EncodeToString([]byte(addr)) + coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + encodedAddr: { + Address: encodedAddr, + Balance: "1000", + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "TTTT-abcd", + Nonce: 2, + Balance: "1000", + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + }, + }, }, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header := &dataBlock.Header{ @@ -212,14 +214,14 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids := []string{"6161616162626262-TTTT-abcd-02"} genericResponse := &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-without-type.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, expectedAccountsESDTWithoutType, string(genericResponse.Docs[0].Source)) time.Sleep(time.Second) @@ -232,7 +234,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) TimeStamp: 5040, } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) pool = &indexer.Pool{ @@ -253,24 +255,24 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids = []string{"TTTT-abcd"} genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, expectedSemiFungibleToken, string(genericResponse.Docs[0].Source)) ids = []string{"6161616162626262-TTTT-abcd-02"} genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-with-type.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, expectedAccountsESDTWithType, string(genericResponse.Docs[0].Source)) ids = []string{"TTTT-abcd-02"} genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token-after-create.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, expectedSemiFungibleTokenAfterCreate, string(genericResponse.Docs[0].Source)) } From dead5604f761bd0bb0256f57dba14a6681fa86df Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 18 May 2022 15:48:43 +0300 Subject: [PATCH 12/96] further fixes for int tests --- .../issueTokenAndSetRoles_test.go | 50 +++---------------- integrationtests/nftIssueCreateBurn_test.go | 11 ++-- integrationtests/updateNFT_test.go | 13 +++-- 3 files changed, 19 insertions(+), 55 deletions(-) diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index fe55ab25..a40d8863 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -22,13 +22,12 @@ func TestIssueTokenAndSetRole(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) body := &dataBlock.Body{} @@ -55,7 +54,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -84,40 +83,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool) - require.Nil(t, err) - - ids = []string{"TOK-abcd"} - genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) - require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) - - // TRANSFER ROLE - pool = &indexer.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: []byte("my-address"), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("false")}, - }, - { - Address: []byte("new-address"), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("true")}, - }, - }, - }, - }, - }, - } - - header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -127,6 +93,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-transfer-role.json"), string(genericResponse.Docs[0].Source)) // UNSET ROLES + pool = &indexer.Pool{ Logs: []*coreData.LogData{ { @@ -146,7 +113,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -162,14 +129,13 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) body := &dataBlock.Body{} @@ -198,7 +164,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -226,7 +192,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 8785912a..d0b6cfea 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -24,7 +24,6 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} // ################ ISSUE NON FUNGIBLE TOKEN ########################## @@ -32,7 +31,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) body := &dataBlock.Body{} @@ -59,7 +58,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{"NON-abcd"} @@ -73,7 +72,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { SelfID: 0, } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header = &dataBlock.Header{ @@ -106,7 +105,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids = []string{"NON-abcd-02"} @@ -140,7 +139,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids = []string{"NON-abcd-02"} diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index 06bf8fc6..793ecdaa 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -31,11 +31,10 @@ func TestNFTUpdateMetadata(t *testing.T) { } marshalizedCreate, _ := json.Marshal(esdtCreateData) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header := &dataBlock.Header{ @@ -62,7 +61,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{"NFT-abcd-0e"} @@ -89,7 +88,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) // Add URIS 2 --- results should be the same @@ -110,7 +109,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) // Update attributes 1 @@ -137,7 +136,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -165,7 +164,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} From 0cd528e35b211b45900521d7a9f520f79c1ebddd Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 18 May 2022 16:12:00 +0300 Subject: [PATCH 13/96] revert some changes --- .../accountsESDTWithTokenType_test.go | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index deec35fc..a31d84c3 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -20,16 +20,6 @@ import ( "github.com/stretchr/testify/require" ) -const ( - expectedTokenAfterIssue = `{"name":"SEMI-token","ticker":"SEM","token":"SEMI-abcd","issuer":"61646472","currentOwner":"61646472","type":"SemiFungibleESDT","timestamp":5040,"ownersHistory":[{"address":"61646472","timestamp":5040}]}` - expectedAccountESDT = `{"address":"6161616162626262","balance":"1000","balanceNum":1e-15,"token":"SEMI-abcd","identifier":"SEMI-abcd-02","tokenNonce":2,"properties":"6f6b","data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"timestamp":5600,"type":"SemiFungibleESDT"}` - - expectedAccountsESDTWithoutType = `{"address":"6161616162626262","balance":"1000","balanceNum":1e-15,"token":"TTTT-abcd","identifier":"TTTT-abcd-02","tokenNonce":2,"properties":"6f6b","data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"timestamp":5600}` - expectedSemiFungibleToken = `{"name":"TTTT-token","ticker":"SEM","token":"TTTT-abcd","issuer":"61646472","currentOwner":"61646472","type":"SemiFungibleESDT","timestamp":5040,"ownersHistory":[{"address":"61646472","timestamp":5040}]}` - expectedAccountsESDTWithType = `{"address":"6161616162626262","balance":"1000","balanceNum":1.0E-15,"token":"TTTT-abcd","identifier":"TTTT-abcd-02","tokenNonce":2,"properties":"6f6b","data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"timestamp":5600,"type":"SemiFungibleESDT"}` - expectedSemiFungibleTokenAfterCreate = `{"identifier":"TTTT-abcd-02","token":"TTTT-abcd","nonce":2,"timestamp":5600,"data":{"creator":"63726561746f72","nonEmptyURIs":false,"whiteListedStorage":false},"type":"SemiFungibleESDT"}` -) - func TestIndexAccountESDTWithTokenType(t *testing.T) { setLogLevelDebug() @@ -77,7 +67,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { genericResponse := &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, expectedTokenAfterIssue, string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/token-after-issue.json"), string(genericResponse.Docs[0].Source)) // ################ CREATE SEMI FUNGIBLE TOKEN ########################## shardCoordinator = &mock.ShardCoordinatorMock{ @@ -143,7 +133,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, expectedAccountESDT, string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt.json"), string(genericResponse.Docs[0].Source)) } @@ -221,7 +211,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) genericResponse := &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, expectedAccountsESDTWithoutType, string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-without-type.json"), string(genericResponse.Docs[0].Source)) time.Sleep(time.Second) @@ -262,17 +252,17 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, expectedSemiFungibleToken, string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token.json"), string(genericResponse.Docs[0].Source)) ids = []string{"6161616162626262-TTTT-abcd-02"} genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, expectedAccountsESDTWithType, string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-with-type.json"), string(genericResponse.Docs[0].Source)) ids = []string{"TTTT-abcd-02"} genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, expectedSemiFungibleTokenAfterCreate, string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token-after-create.json"), string(genericResponse.Docs[0].Source)) } From 815db673a4631bc57d60aef46ea9096216a14443 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 18 May 2022 16:37:30 +0300 Subject: [PATCH 14/96] final int tests fixes (hopefully) --- .../issueTokenAndSetRoles_test.go | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index a40d8863..e1013c4b 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -13,6 +13,7 @@ import ( dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -83,7 +84,40 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, nil) + require.Nil(t, err) + + ids = []string{"TOK-abcd"} + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + require.Nil(t, err) + assert.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) + + // TRANSFER ROLE + pool = &indexer.Pool{ + Logs: []*coreData.LogData{ + { + TxHash: "h1", + LogHandler: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: []byte("my-address"), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("false")}, + }, + { + Address: []byte("new-address"), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("true")}, + }, + }, + }, + }, + }, + } + + header.TimeStamp = 10000 + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -93,7 +127,6 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-transfer-role.json"), string(genericResponse.Docs[0].Source)) // UNSET ROLES - pool = &indexer.Pool{ Logs: []*coreData.LogData{ { @@ -171,7 +204,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { genericResponse := &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) + assert.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) // ISSUE pool = &indexer.Pool{ From d36c936db4638d7c74d26fced7f534170fb3568e Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Mon, 30 May 2022 16:56:55 +0300 Subject: [PATCH 15/96] fixed integration tests --- .../accountsBalanceNftTransfer_test.go | 88 ++++++++----------- integrationtests/createNFTWithTags_test.go | 42 +++++---- 2 files changed, 61 insertions(+), 69 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 3b6c0548..78c6c442 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -3,8 +3,7 @@ package integrationtests import ( - "bytes" - "encoding/json" + "encoding/hex" "math/big" "testing" @@ -13,10 +12,8 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) @@ -26,7 +23,6 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} // ################ CREATE NFT ########################## @@ -36,25 +32,10 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { body := &dataBlock.Body{} - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - } - addr := "test-address-balance-1" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, - } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + addrHex := hex.EncodeToString([]byte(addr)) + + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header := &dataBlock.Header{ @@ -80,7 +61,20 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + addrHex: { + Address: addrHex, + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "NFT-abcdef", + Nonce: 7440483, + Balance: "1000", + }, + }, + }, + } + + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids := []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} @@ -114,38 +108,28 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - esdtToken = &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - } addrReceiver := "new-address" - mockAccountReceiver := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addrReceiver) - }, - } - mockAccountSender := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return []byte("{}"), nil - }, - AddressBytesCalled: func() []byte { - return []byte(addr) + addrReceiverHex := hex.EncodeToString([]byte(addrReceiver)) + + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + require.Nil(t, err) + + coreAlteredAccounts = map[string]*indexer.AlteredAccount{ + addrHex: { + Address: addrHex, }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - if bytes.Equal(container, []byte(addr)) { - return mockAccountSender, nil - } - return mockAccountReceiver, nil + addrReceiverHex: { + Address: addrReceiverHex, + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "NFT-abcdef", + Nonce: 7440483, + Balance: "1000", + }, + }, }, } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) - require.Nil(t, err) - - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 62b641cf..fddd5908 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "math/big" "testing" @@ -15,7 +16,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) @@ -45,20 +45,9 @@ func TestCreateNFTWithTags(t *testing.T) { } addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts := &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, - } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + addrHex := hex.EncodeToString([]byte(addr)) + + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) esdtDataBytes, _ := json.Marshal(esdtToken) @@ -82,8 +71,26 @@ func TestCreateNFTWithTags(t *testing.T) { }, } + coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + addrHex: { + Address: addrHex, + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "DESK-abcd", + Nonce: 1, + Balance: "1000", + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + Attributes: []byte("tags:hello,something,ceva,an,so,on;metadata:QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud"), + }, + }, + }, + }, + } + body := &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids := []string{"6161616162626262-DESK-abcd-01"} @@ -125,8 +132,9 @@ func TestCreateNFTWithTags(t *testing.T) { }, } + coreAlteredAccounts[addrHex].Tokens[0].Nonce = 2 body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = []string{"aGVsbG8=", "Y2V2YQ==", "YW4=", "b24=", "c28=", "c28="} From 37f50aab58b5432f1c520333d46e95c49bf5c55e Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Fri, 3 Jun 2022 10:58:07 +0300 Subject: [PATCH 16/96] fixes after review --- integrationtests/issueTokenAndSetRoles_test.go | 5 ++--- process/accounts/accountsProcessor.go | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 2ea7721a..9bbcd176 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -13,7 +13,6 @@ import ( dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -96,7 +95,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { genericResponse = &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - assert.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) // TRANSFER ROLE pool = &indexer.Pool{ @@ -209,7 +208,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { genericResponse := &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) - assert.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) // ISSUE pool = &indexer.Pool{ diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index 3754b8e6..543267c6 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -42,6 +42,7 @@ func NewAccountsProcessor( }, nil } +// TODO: refactor this as the altered accounts are already computed on the node. EN-12389 // GetAccounts will get accounts for regular operations and esdt operations func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) From dd5be1f5e05c13627fc1b83c5342f342f80e0e27 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Thu, 7 Jul 2022 14:53:18 +0300 Subject: [PATCH 17/96] fix integration tests --- integrationtests/collectionsIndex_test.go | 109 ++++++++++++--------- integrationtests/createNFTWithTags_test.go | 31 ++---- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index a09f3afa..b58432e8 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "math/big" "testing" @@ -15,7 +16,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) @@ -30,9 +30,8 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { SelfID: core.MetachainShardId, } - accounts := &mock.AccountsStub{} feeComputer := &mock.EconomicsHandlerMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) body := &dataBlock.Body{} @@ -59,7 +58,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) // ################ CREATE SEMI FUNGIBLE TOKEN 1 ########################## @@ -67,29 +66,45 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { SelfID: 0, } - esdtToken := &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - Properties: []byte("ok"), - TokenMetaData: &esdt.MetaData{ - Creator: []byte("creator"), - }, - } - addr := "aaaabbbbcccccccc" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) + addrHex := hex.EncodeToString([]byte(addr)) + + addrForLog := "aaaabbbb" + addrForLogHex := hex.EncodeToString([]byte(addrForLog)) + + coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + addrHex: { + Address: addrHex, + Balance: "0", + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "SSSS-dddd", + Balance: "1000", + Nonce: 2, + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + }, + }, }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil + addrForLogHex: { + Address: addrForLogHex, + Balance: "0", + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "SSSS-dddd", + Balance: "1000", + Nonce: 2, + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + }, + }, }, } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) header = &dataBlock.Header{ @@ -111,7 +126,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { LogHandler: &transaction.Log{ Events: []*transaction.Event{ { - Address: []byte("aaaabbbb"), + Address: []byte(addr), Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), Topics: [][]byte{[]byte("SSSS-dddd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, @@ -122,7 +137,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids := []string{"61616161626262626363636363636363"} genericResponse := &GenericResponse{} @@ -138,7 +153,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { LogHandler: &transaction.Log{ Events: []*transaction.Event{ { - Address: []byte("aaaabbbb"), + Address: []byte(addr), Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), Topics: [][]byte{[]byte("SSSS-dddd"), big.NewInt(22).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, @@ -149,7 +164,10 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + coreAlteredAccounts[addrHex].Tokens[0].Nonce = 22 + coreAlteredAccounts[addrForLogHex].Tokens[0].Nonce = 22 + + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} @@ -158,29 +176,26 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/collectionsIndex/collections-2.json"), string(genericResponse.Docs[0].Source)) // ################ TRANSFER SEMI FUNGIBLE TOKEN 2 ########################## - esdtToken = &esdt.ESDigitalToken{ - Value: big.NewInt(0), - Properties: []byte("ok"), - TokenMetaData: &esdt.MetaData{ - Creator: []byte("creator"), - }, - } addr = "aaaabbbbcccccccc" - mockAccount = &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil + addrHex = hex.EncodeToString([]byte(addr)) + coreAlteredAccounts = map[string]*indexer.AlteredAccount{ + addrHex: { + Address: addrHex, + Tokens: []*indexer.AccountTokenData{ + { + Identifier: "NFT-abcdef", + Balance: "0", + Properties: "ok", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + }, + }, }, } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) pool = &indexer.Pool{ @@ -201,7 +216,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 199a60ac..ac2588a0 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -74,6 +74,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts := map[string]*indexer.AlteredAccount{ addrHex: { Address: addrHex, + Balance: "0", Tokens: []*indexer.AccountTokenData{ { Identifier: "DESK-abcd", @@ -82,7 +83,7 @@ func TestCreateNFTWithTags(t *testing.T) { Properties: "ok", MetaData: &esdt.MetaData{ Creator: []byte("creator"), - Attributes: []byte("tags:hello,something,ceva,an,so,on;metadata:QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud"), + Attributes: []byte("tags:hello,something,do,music,art,gallery;metadata:QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud"), }, }, }, @@ -160,28 +161,10 @@ func TestCreateNFTWithTags(t *testing.T) { hexEncodedAttributes := "746167733a5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c2c3c3c3c3e3e3e2626262626262626262626262626262c272727273b6d657461646174613a516d533757525566464464516458654c513637516942394a33663746654d69343554526d6f79415741563568345a" attributes, _ := hex.DecodeString(hexEncodedAttributes) - esdtToken = &esdt.ESDigitalToken{ - Value: big.NewInt(1000), - Properties: []byte("ok"), - TokenMetaData: &esdt.MetaData{ - Creator: []byte("creator"), - Attributes: attributes, - }, - } - mockAccount = &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts = &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil - }, - } - esProc, err = CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + coreAlteredAccounts[addrHex].Tokens[0].Nonce = 3 + coreAlteredAccounts[addrHex].Tokens[0].MetaData.Attributes = attributes + + esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) pool = &indexer.Pool{ @@ -203,7 +186,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") From ba6de83379e994bcc8777410974302efbdc4d013 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 27 Jul 2022 15:25:02 +0300 Subject: [PATCH 18/96] preparations remove TransactionFeeHandler --- go.mod | 2 +- go.sum | 4 +- process/elasticProcessor_test.go | 16 +-- .../smartContractResultsProcessor.go | 6 +- process/transactions/transactionsGrouper.go | 12 +- .../transactions/transactionsGrouper_test.go | 22 +-- .../transactionsProcessor_test.go | 128 +++++++++--------- workItems/workItemBlock.go | 37 +++-- workItems/workItemBlock_test.go | 8 +- 9 files changed, 125 insertions(+), 110 deletions(-) diff --git a/go.mod b/go.mod index 72762edb..db185f3f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220414130405-e3cc29bc7711 + github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727115746-10577564b31b github.com/ElrondNetwork/elrond-go-logger v1.0.7 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 2591cd8e..4f325984 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220414130405-e3cc29bc7711 h1:pU3ZyHL/gMg/2cN+DxG3tpalVT+iJfKysE6S7GwzB4Y= -github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220414130405-e3cc29bc7711/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727115746-10577564b31b h1:pDKUE6Zy6T57heCQJeeav6fcFJaRUfRsqIFqLpzbZfA= +github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727115746-10577564b31b/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7 h1:Ldl1rVS0RGKc1IsW8jIaGCb6Zwei04gsMvyjL05X6mE= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/process/elasticProcessor_test.go b/process/elasticProcessor_test.go index 95904faa..f67c0b2e 100644 --- a/process/elasticProcessor_test.go +++ b/process/elasticProcessor_test.go @@ -81,9 +81,9 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { } } -func newTestTxPool() map[string]coreData.TransactionHandler { - txPool := map[string]coreData.TransactionHandler{ - "tx1": &transaction.Transaction{ +func newTestTxPool() map[string]coreData.TransactionHandlerWithGasUsedAndFee { + txPool := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "tx1": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: uint64(1), Value: big.NewInt(1), RcvAddr: []byte("receiver_address1"), @@ -92,8 +92,8 @@ func newTestTxPool() map[string]coreData.TransactionHandler { GasLimit: uint64(1000), Data: []byte("tx_data1"), Signature: []byte("signature1"), - }, - "tx2": &transaction.Transaction{ + }, 0, big.NewInt(0)), + "tx2": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: uint64(2), Value: big.NewInt(2), RcvAddr: []byte("receiver_address2"), @@ -102,8 +102,8 @@ func newTestTxPool() map[string]coreData.TransactionHandler { GasLimit: uint64(1000), Data: []byte("tx_data2"), Signature: []byte("signature2"), - }, - "tx3": &transaction.Transaction{ + }, 0, big.NewInt(0)), + "tx3": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: uint64(3), Value: big.NewInt(3), RcvAddr: []byte("receiver_address3"), @@ -112,7 +112,7 @@ func newTestTxPool() map[string]coreData.TransactionHandler { GasLimit: uint64(1000), Data: []byte("tx_data3"), Signature: []byte("signature3"), - }, + }, 0, big.NewInt(0)), } return txPool diff --git a/process/transactions/smartContractResultsProcessor.go b/process/transactions/smartContractResultsProcessor.go index 0cee7e2d..13633b42 100644 --- a/process/transactions/smartContractResultsProcessor.go +++ b/process/transactions/smartContractResultsProcessor.go @@ -44,7 +44,7 @@ func newSmartContractResultsProcessor( func (proc *smartContractResultsProcessor) processSCRs( body *block.Body, header coreData.HeaderHandler, - txsHandler map[string]data.TransactionHandler, + txsHandler map[string]data.TransactionHandlerWithGasUsedAndFee, ) []*indexerData.ScResult { allSCRs := make([]*indexerData.ScResult, 0, len(txsHandler)) scrs := convertHandlerMap(txsHandler) @@ -68,10 +68,10 @@ func (proc *smartContractResultsProcessor) processSCRs( return allSCRs } -func convertHandlerMap(txsHandler map[string]data.TransactionHandler) map[string]*smartContractResult.SmartContractResult { +func convertHandlerMap(txsHandler map[string]data.TransactionHandlerWithGasUsedAndFee) map[string]*smartContractResult.SmartContractResult { scrs := make(map[string]*smartContractResult.SmartContractResult, len(txsHandler)) for txHandlerHash, txHandler := range txsHandler { - scr, ok := txHandler.(*smartContractResult.SmartContractResult) + scr, ok := txHandler.GetTxHandler().(*smartContractResult.SmartContractResult) if !ok { log.Warn("smartContractResultsProcessor.processSCRsFromMiniblock cannot convert TransactionHandler to scr", "scr hash", hex.EncodeToString([]byte(txHandlerHash)), diff --git a/process/transactions/transactionsGrouper.go b/process/transactions/transactionsGrouper.go index f8968586..3187cf35 100644 --- a/process/transactions/transactionsGrouper.go +++ b/process/transactions/transactionsGrouper.go @@ -44,7 +44,7 @@ func (tg *txsGrouper) groupNormalTxs( mbIndex int, mb *block.MiniBlock, header coreData.HeaderHandler, - txs map[string]coreData.TransactionHandler, + txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, alteredAccounts data.AlteredAccountsHandler, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) @@ -97,7 +97,7 @@ func (tg *txsGrouper) prepareNormalTxForDB( mb *block.MiniBlock, mbStatus string, txHash []byte, - txs map[string]coreData.TransactionHandler, + txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, header coreData.HeaderHandler, ) (*data.Transaction, bool) { txHandler, okGet := txs[string(txHash)] @@ -105,7 +105,7 @@ func (tg *txsGrouper) prepareNormalTxForDB( return nil, false } - tx, okCast := txHandler.(*transaction.Transaction) + tx, okCast := txHandler.GetTxHandler().(*transaction.Transaction) if !okCast { return nil, false } @@ -172,7 +172,7 @@ func (tg *txsGrouper) groupInvalidTxs( mbIndex int, mb *block.MiniBlock, header coreData.HeaderHandler, - txs map[string]coreData.TransactionHandler, + txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, alteredAccounts data.AlteredAccountsHandler, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) @@ -199,7 +199,7 @@ func (tg *txsGrouper) prepareInvalidTxForDB( mbHash []byte, mb *block.MiniBlock, txHash []byte, - txs map[string]coreData.TransactionHandler, + txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, header coreData.HeaderHandler, ) (*data.Transaction, bool) { txHandler, okGet := txs[string(txHash)] @@ -207,7 +207,7 @@ func (tg *txsGrouper) prepareInvalidTxForDB( return nil, false } - tx, okCast := txHandler.(*transaction.Transaction) + tx, okCast := txHandler.GetTxHandler().(*transaction.Transaction) if !okCast { return nil, false } diff --git a/process/transactions/transactionsGrouper_test.go b/process/transactions/transactionsGrouper_test.go index 9fce1b45..52c4545e 100644 --- a/process/transactions/transactionsGrouper_test.go +++ b/process/transactions/transactionsGrouper_test.go @@ -1,6 +1,8 @@ package transactions import ( + "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "math/big" "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" @@ -27,15 +29,15 @@ func TestGroupNormalTxs(t *testing.T) { Type: block.TxBlock, } header := &block.Header{} - txs := map[string]coreData.TransactionHandler{ - string(txHash1): &transaction.Transaction{ + txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender1"), RcvAddr: []byte("receiver1"), - }, - string(txHash2): &transaction.Transaction{ + }, 0, big.NewInt(0)), + string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender2"), RcvAddr: []byte("receiver2"), - }, + }, 0, big.NewInt(0)), } alteredAddresses := data.NewAlteredAccounts() @@ -87,15 +89,15 @@ func TestGroupInvalidTxs(t *testing.T) { Type: block.InvalidBlock, } header := &block.Header{} - txs := map[string]coreData.TransactionHandler{ - string(txHash1): &transaction.Transaction{ + txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender1"), RcvAddr: []byte("receiver1"), - }, - string(txHash2): &transaction.Transaction{ + }, 0, big.NewInt(0)), + string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender2"), RcvAddr: []byte("receiver2"), - }, + }, 0, big.NewInt(0)), } alteredAddresses := data.NewAlteredAccounts() diff --git a/process/transactions/transactionsProcessor_test.go b/process/transactions/transactionsProcessor_test.go index a78bfa97..4b01c517 100644 --- a/process/transactions/transactionsProcessor_test.go +++ b/process/transactions/transactionsProcessor_test.go @@ -92,21 +92,21 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { t.Parallel() txHash1 := []byte("txHash1") - tx1 := &transaction.Transaction{ + tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, - } + }, 0, big.NewInt(0)) txHash2 := []byte("txHash2") - tx2 := &transaction.Transaction{ + tx2 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, - } + }, 0, big.NewInt(0)) txHash3 := []byte("txHash3") - tx3 := &transaction.Transaction{} + tx3 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) txHash4 := []byte("txHash4") - tx4 := &transaction.Transaction{} + tx4 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) txHash5 := []byte("txHash5") - tx5 := &transaction.Transaction{} + tx5 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) rTx1Hash := []byte("rTxHash1") rTx1 := &rewardTx.RewardTx{} @@ -125,22 +125,22 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { } scHash1 := []byte("scHash1") - scResult1 := &smartContractResult.SmartContractResult{ + scResult1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, - } + }, 0, big.NewInt(0)) scHash2 := []byte("scHash2") - scResult2 := &smartContractResult.SmartContractResult{ + scResult2 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, - } + }, 0, big.NewInt(0)) scHash3 := []byte("scHash3") - scResult3 := &smartContractResult.SmartContractResult{ + scResult3 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash3, Data: []byte("@" + "6F6B"), - } + }, 0, big.NewInt(0)) body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -177,13 +177,13 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { header := &block.Header{} pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, string(txHash2): tx2, string(txHash3): tx3, string(txHash4): tx4, }, - Scrs: map[string]coreData.TransactionHandler{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scHash1): scResult1, string(scHash2): scResult2, string(scHash3): scResult3, @@ -192,7 +192,7 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { string(rTx1Hash): rTx1, string(rTx2Hash): rTx2, }, - Invalid: map[string]coreData.TransactionHandler{ + Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash5): tx5, }, Receipts: map[string]coreData.TransactionHandler{ @@ -212,24 +212,24 @@ func TestRelayedTransactions(t *testing.T) { t.Parallel() txHash1 := []byte("txHash1") - tx1 := &transaction.Transaction{ + tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, Data: []byte("relayedTx@blablabllablalba"), - } + }, 0, big.NewInt(0)) scHash1 := []byte("scHash1") - scResult1 := &smartContractResult.SmartContractResult{ + scResult1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, - } + }, 0, big.NewInt(0)) scHash2 := []byte("scHash2") - scResult2 := &smartContractResult.SmartContractResult{ + scResult2 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, - } + }, 0, big.NewInt(0)) body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -247,10 +247,10 @@ func TestRelayedTransactions(t *testing.T) { header := &block.Header{} pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, - Scrs: map[string]coreData.TransactionHandler{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scHash1): scResult1, string(scHash2): scResult2, }, @@ -300,19 +300,19 @@ func TestAlteredAddresses(t *testing.T) { address1 := []byte("address1") // should be added address2 := []byte("address2") expectedAlteredAccounts[hex.EncodeToString(address1)] = struct{}{} - tx1 := &transaction.Transaction{ + tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: address1, RcvAddr: address2, - } + }, 0, big.NewInt(0)) tx1Hash := []byte("tx1Hash") address3 := []byte("address3") address4 := []byte("address4") // should be added expectedAlteredAccounts[hex.EncodeToString(address4)] = struct{}{} - tx2 := &transaction.Transaction{ + tx2 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: address3, RcvAddr: address4, - } + }, 0, big.NewInt(0)) tx2Hash := []byte("tx2hash") txMiniBlock1 := &block.MiniBlock{ @@ -359,19 +359,19 @@ func TestAlteredAddresses(t *testing.T) { address7 := []byte("address7") // should be added address8 := []byte("address8") expectedAlteredAccounts[hex.EncodeToString(address7)] = struct{}{} - scr1 := &smartContractResult.SmartContractResult{ + scr1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ RcvAddr: address7, SndAddr: address8, - } + }, 0, big.NewInt(0)) scr1Hash := []byte("scr1Hash") address9 := []byte("address9") // should be added address10 := []byte("address10") expectedAlteredAccounts[hex.EncodeToString(address9)] = struct{}{} - scr2 := &smartContractResult.SmartContractResult{ + scr2 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ RcvAddr: address9, SndAddr: address10, - } + }, 0, big.NewInt(0)) scr2Hash := []byte("scr2Hash") scrMiniBlock1 := &block.MiniBlock{ @@ -388,11 +388,11 @@ func TestAlteredAddresses(t *testing.T) { hdr := &block.Header{} pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(tx1Hash): tx1, string(tx2Hash): tx2, }, - Scrs: map[string]coreData.TransactionHandler{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scr1Hash): scr1, string(scr2Hash): scr2, }, @@ -465,10 +465,10 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) txHash1 := []byte("txHash1") - tx1 := &transaction.Transaction{ + tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, - } + }, 0, big.NewInt(0)) recHash1 := []byte("recHash1") rec1 := &receipt.Receipt{ Value: big.NewInt(100), @@ -491,7 +491,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { header := &block.Header{} pool := &indexer.Pool{ - Invalid: map[string]coreData.TransactionHandler{ + Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, Receipts: map[string]coreData.TransactionHandler{ @@ -501,7 +501,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { results := txDbProc.PrepareTransactionsForDatabase(body, header, pool) require.Len(t, results.Transactions, 1) - require.Equal(t, tx1.GasLimit, results.Transactions[0].GasUsed) + require.Equal(t, tx1.GetGasLimit(), results.Transactions[0].GasUsed) } func TestGetRewardsTxsHashesHexEncoded(t *testing.T) { @@ -554,15 +554,15 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) txHash1 := []byte("txHash1") - tx1 := &transaction.Transaction{ + tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 123456, Data: []byte("ESDTTransfer@54474e2d383862383366@0a"), - } + }, 0, big.NewInt(0)) scResHash1 := []byte("scResHash1") - scRes1 := &smartContractResult.SmartContractResult{ + scRes1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, - } + }, 0, big.NewInt(0)) body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -580,10 +580,10 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * header := &block.Header{} pool := &indexer.Pool{ - Invalid: map[string]coreData.TransactionHandler{ + Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, - Scrs: map[string]coreData.TransactionHandler{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scResHash1): scRes1, }, } @@ -609,18 +609,18 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }} txHash1 := []byte("txHash1") - tx1 := &transaction.Transaction{ + tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 123456, Data: []byte("ESDTNFTTransfer@595959453643392D303837363661@01@01@000000000000000005005C83E0C42EDCE394F40B24D29D298B0249C41F028974@66756E64@890479AFC610F4BEBC087D3ADA3F7C2775C736BBA91F41FD3D65092AA482D8B0@1c20"), - } + }, 0, big.NewInt(0)) scResHash1 := []byte("scResHash1") - scRes1 := &smartContractResult.SmartContractResult{ + scRes1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 1, Data: []byte("@" + okHexEncoded), OriginalTxHash: txHash1, PrevTxHash: txHash1, - } + }, 0, big.NewInt(0)) body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -638,10 +638,10 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t header := &block.Header{} pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, - Scrs: map[string]coreData.TransactionHandler{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scResHash1): scRes1, }, } @@ -688,26 +688,26 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { } header := &block.Header{} pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - "t1": &transaction.Transaction{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "t1": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - "scr1": &smartContractResult.SmartContractResult{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "scr1": indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("ESDTTransfer@414141414141412d323436626461@0186a0"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }, - "scr2": &smartContractResult.SmartContractResult{ + }, 0, big.NewInt(0)), + "scr2": indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("@6f6b"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }, + }, 0, big.NewInt(0)), }, } @@ -720,20 +720,20 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { // transaction fail pool = &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - "t1": &transaction.Transaction{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "t1": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - "scr1": &smartContractResult.SmartContractResult{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "scr1": indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("75736572206572726f72"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }, + }, 0, big.NewInt(0)), }, } diff --git a/workItems/workItemBlock.go b/workItems/workItemBlock.go index 02b7be73..4b87d9bf 100644 --- a/workItems/workItemBlock.go +++ b/workItems/workItemBlock.go @@ -103,26 +103,39 @@ func (wib *itemBlock) IsInterfaceNil() bool { // ComputeSizeOfTxs will compute size of transactions in bytes func ComputeSizeOfTxs(marshalizer marshal.Marshalizer, pool *indexer.Pool) int { sizeTxs := 0 - sizeTxs += computeSizeOfMap(marshalizer, pool.Txs) - sizeTxs += computeSizeOfMap(marshalizer, pool.Receipts) - sizeTxs += computeSizeOfMap(marshalizer, pool.Invalid) - sizeTxs += computeSizeOfMap(marshalizer, pool.Rewards) - sizeTxs += computeSizeOfMap(marshalizer, pool.Scrs) + sizeTxs += computeSizeOfMapTxsWithGasUsedAndFee(marshalizer, pool.Txs) + sizeTxs += computeSizeOfMapTxsWithGasUsedAndFee(marshalizer, pool.Scrs) + sizeTxs += computeSizeOfMapTxsWithGasUsedAndFee(marshalizer, pool.Invalid) + sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Rewards) + sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Receipts) return sizeTxs } -func computeSizeOfMap(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandler) int { +func computeSizeOfMapTxsWithGasUsedAndFee(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandlerWithGasUsedAndFee) int { txsSize := 0 for _, tx := range mapTxs { - txBytes, err := marshalizer.Marshal(tx) - if err != nil { - log.Debug("itemBlock.computeSizeOfMap", "error", err) - continue - } + txsSize += computeTxSize(marshalizer, tx.GetTxHandler()) + } + + return txsSize +} - txsSize += len(txBytes) +func computeSizeOfMapTxs(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandler) int { + txsSize := 0 + for _, tx := range mapTxs { + txsSize += computeTxSize(marshalizer, tx) } return txsSize } + +func computeTxSize(marshalizer marshal.Marshalizer, tx data.TransactionHandler) int { + txBytes, err := marshalizer.Marshal(tx) + if err != nil { + log.Debug("itemBlock.computeTxSize", "error", err) + return 0 + } + + return len(txBytes) +} diff --git a/workItems/workItemBlock_test.go b/workItems/workItemBlock_test.go index 812851e2..8df1b532 100644 --- a/workItems/workItemBlock_test.go +++ b/workItems/workItemBlock_test.go @@ -19,8 +19,8 @@ import ( "github.com/stretchr/testify/require" ) -func generateTxs(numTxs int) map[string]data.TransactionHandler { - txs := make(map[string]data.TransactionHandler, numTxs) +func generateTxs(numTxs int) map[string]data.TransactionHandlerWithGasUsedAndFee { + txs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, numTxs) for i := 0; i < numTxs; i++ { tx := &transaction.Transaction{ Nonce: uint64(i), @@ -32,7 +32,7 @@ func generateTxs(numTxs int) map[string]data.TransactionHandler { Data: []byte("dasjdksakjdksajdjksajkdjkasjdksajkdasjdksakjdksajdjksajkdjkasjdksajkdasjdksakjdksajdjksajkdjkasjdksajk"), Signature: []byte("randomSignatureasdasldkasdsahjgdlhjaskldsjkaldjklasjkdjskladjkl;sajkl"), } - txs[fmt.Sprintf("%d", i)] = tx + txs[fmt.Sprintf("%d", i)] = indexer.NewTransactionHandlerWithGasAndFee(tx, 0, big.NewInt(0)) } return txs @@ -200,7 +200,7 @@ func TestComputeSizeOfTxs(t *testing.T) { lenTxs := workItems.ComputeSizeOfTxs(gogoMarsh, &indexer.Pool{Txs: txs}) keys := reflect.ValueOf(txs).MapKeys() - oneTxBytes, _ := gogoMarsh.Marshal(txs[keys[0].String()]) + oneTxBytes, _ := gogoMarsh.Marshal(txs[keys[0].String()].GetTxHandler()) oneTxSize := len(oneTxBytes) expectedSize := numTxs * oneTxSize expectedSizeDeltaPlus := expectedSize + int(0.01*float64(expectedSize)) From 816878ff95ffb110a1e92789bc551e074142f134 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 27 Jul 2022 15:25:23 +0300 Subject: [PATCH 19/96] fix import --- process/transactions/transactionsGrouper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/transactions/transactionsGrouper_test.go b/process/transactions/transactionsGrouper_test.go index 52c4545e..71408ac0 100644 --- a/process/transactions/transactionsGrouper_test.go +++ b/process/transactions/transactionsGrouper_test.go @@ -1,7 +1,6 @@ package transactions import ( - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "math/big" "testing" @@ -9,6 +8,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/receipt" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" "github.com/ElrondNetwork/elrond-go-core/data/transaction" From 0acb630d84c3a10c11ab229825fb848d1eb9a97c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 28 Jul 2022 08:42:27 +0300 Subject: [PATCH 20/96] fixes --- go.mod | 2 +- go.sum | 4 +-- process/transactions/transactionsGrouper.go | 10 +++---- .../transactions/transactionsGrouper_test.go | 20 ++++++------- .../transactionsProcessor_test.go | 28 +++++++++---------- workItems/workItemBlock.go | 17 +++-------- 6 files changed, 36 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index db185f3f..b276231e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727115746-10577564b31b + github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727124042-48330ce8d452 github.com/ElrondNetwork/elrond-go-logger v1.0.7 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 4f325984..e9e9af55 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727115746-10577564b31b h1:pDKUE6Zy6T57heCQJeeav6fcFJaRUfRsqIFqLpzbZfA= -github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727115746-10577564b31b/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727124042-48330ce8d452 h1:xYVzM0FOxJrCOl8kuHT0Jy6ZfjEJakawVSCqE11QgDc= +github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727124042-48330ce8d452/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7 h1:Ldl1rVS0RGKc1IsW8jIaGCb6Zwei04gsMvyjL05X6mE= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/process/transactions/transactionsGrouper.go b/process/transactions/transactionsGrouper.go index 3187cf35..bcb79e71 100644 --- a/process/transactions/transactionsGrouper.go +++ b/process/transactions/transactionsGrouper.go @@ -119,7 +119,7 @@ func (tg *txsGrouper) groupRewardsTxs( mbIndex int, mb *block.MiniBlock, header coreData.HeaderHandler, - txs map[string]coreData.TransactionHandler, + txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, alteredAccounts data.AlteredAccountsHandler, ) (map[string]*data.Transaction, error) { rewardsTxs := make(map[string]*data.Transaction) @@ -150,7 +150,7 @@ func (tg *txsGrouper) prepareRewardTxForDB( mb *block.MiniBlock, mbStatus string, txHash []byte, - txs map[string]coreData.TransactionHandler, + txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, header coreData.HeaderHandler, ) (*data.Transaction, bool) { txHandler, okGet := txs[string(txHash)] @@ -158,7 +158,7 @@ func (tg *txsGrouper) prepareRewardTxForDB( return nil, false } - rtx, okCast := txHandler.(*rewardTx.RewardTx) + rtx, okCast := txHandler.GetTxHandler().(*rewardTx.RewardTx) if !okCast { return nil, false } @@ -229,10 +229,10 @@ func (tg *txsGrouper) shouldIndex(destinationShardID uint32) bool { return tg.selfShardID == destinationShardID } -func (tg *txsGrouper) groupReceipts(header coreData.HeaderHandler, txsPool map[string]coreData.TransactionHandler) []*data.Receipt { +func (tg *txsGrouper) groupReceipts(header coreData.HeaderHandler, txsPool map[string]coreData.TransactionHandlerWithGasUsedAndFee) []*data.Receipt { dbReceipts := make([]*data.Receipt, 0) for hash, tx := range txsPool { - rec, ok := tx.(*receipt.Receipt) + rec, ok := tx.GetTxHandler().(*receipt.Receipt) if !ok { continue } diff --git a/process/transactions/transactionsGrouper_test.go b/process/transactions/transactionsGrouper_test.go index 71408ac0..f2346b36 100644 --- a/process/transactions/transactionsGrouper_test.go +++ b/process/transactions/transactionsGrouper_test.go @@ -60,13 +60,13 @@ func TestGroupRewardsTxs(t *testing.T) { Type: block.RewardsBlock, } header := &block.Header{} - txs := map[string]coreData.TransactionHandler{ - string(txHash1): &rewardTx.RewardTx{ + txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ RcvAddr: []byte("receiver1"), - }, - string(txHash2): &rewardTx.RewardTx{ + }, 0, big.NewInt(0)), + string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ RcvAddr: []byte("receiver2"), - }, + }, 0, big.NewInt(0)), } alteredAddresses := data.NewAlteredAccounts() @@ -116,13 +116,13 @@ func TestGroupReceipts(t *testing.T) { txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") header := &block.Header{} - txs := map[string]coreData.TransactionHandler{ - string(txHash1): &receipt.Receipt{ + txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ SndAddr: []byte("sender1"), - }, - string(txHash2): &receipt.Receipt{ + }, 0, big.NewInt(0)), + string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ SndAddr: []byte("sender2"), - }, + }, 0, big.NewInt(0)), } normalTxs := grouper.groupReceipts(header, txs) diff --git a/process/transactions/transactionsProcessor_test.go b/process/transactions/transactionsProcessor_test.go index 4b01c517..db9836f7 100644 --- a/process/transactions/transactionsProcessor_test.go +++ b/process/transactions/transactionsProcessor_test.go @@ -109,20 +109,20 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { tx5 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) rTx1Hash := []byte("rTxHash1") - rTx1 := &rewardTx.RewardTx{} + rTx1 := indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) rTx2Hash := []byte("rTxHash2") - rTx2 := &rewardTx.RewardTx{} + rTx2 := indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) recHash1 := []byte("recHash1") - rec1 := &receipt.Receipt{ + rec1 := indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ Value: big.NewInt(100), TxHash: txHash1, - } + }, 0, big.NewInt(0)) recHash2 := []byte("recHash2") - rec2 := &receipt.Receipt{ + rec2 := indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ Value: big.NewInt(200), TxHash: txHash2, - } + }, 0, big.NewInt(0)) scHash1 := []byte("scHash1") scResult1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ @@ -188,14 +188,14 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { string(scHash2): scResult2, string(scHash3): scResult3, }, - Rewards: map[string]coreData.TransactionHandler{ + Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(rTx1Hash): rTx1, string(rTx2Hash): rTx2, }, Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash5): tx5, }, - Receipts: map[string]coreData.TransactionHandler{ + Receipts: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(recHash1): rec1, string(recHash2): rec2, }, @@ -396,9 +396,9 @@ func TestAlteredAddresses(t *testing.T) { string(scr1Hash): scr1, string(scr2Hash): scr2, }, - Rewards: map[string]coreData.TransactionHandler{ - string(rwdTx1Hash): rwdTx1, - string(rwdTx2Hash): rwdTx2, + Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(rwdTx1Hash): indexer.NewTransactionHandlerWithGasAndFee(rwdTx1, 0, big.NewInt(0)), + string(rwdTx2Hash): indexer.NewTransactionHandlerWithGasAndFee(rwdTx2, 0, big.NewInt(0)), }, } @@ -470,10 +470,10 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { GasPrice: 100, }, 0, big.NewInt(0)) recHash1 := []byte("recHash1") - rec1 := &receipt.Receipt{ + rec1 := indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ Value: big.NewInt(100), TxHash: txHash1, - } + }, 0, big.NewInt(0)) body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -494,7 +494,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, - Receipts: map[string]coreData.TransactionHandler{ + Receipts: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(recHash1): rec1, }, } diff --git a/workItems/workItemBlock.go b/workItems/workItemBlock.go index 4b87d9bf..fa83aa3a 100644 --- a/workItems/workItemBlock.go +++ b/workItems/workItemBlock.go @@ -103,16 +103,16 @@ func (wib *itemBlock) IsInterfaceNil() bool { // ComputeSizeOfTxs will compute size of transactions in bytes func ComputeSizeOfTxs(marshalizer marshal.Marshalizer, pool *indexer.Pool) int { sizeTxs := 0 - sizeTxs += computeSizeOfMapTxsWithGasUsedAndFee(marshalizer, pool.Txs) - sizeTxs += computeSizeOfMapTxsWithGasUsedAndFee(marshalizer, pool.Scrs) - sizeTxs += computeSizeOfMapTxsWithGasUsedAndFee(marshalizer, pool.Invalid) + sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Txs) + sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Scrs) + sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Invalid) sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Rewards) sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Receipts) return sizeTxs } -func computeSizeOfMapTxsWithGasUsedAndFee(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandlerWithGasUsedAndFee) int { +func computeSizeOfMapTxs(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandlerWithGasUsedAndFee) int { txsSize := 0 for _, tx := range mapTxs { txsSize += computeTxSize(marshalizer, tx.GetTxHandler()) @@ -121,15 +121,6 @@ func computeSizeOfMapTxsWithGasUsedAndFee(marshalizer marshal.Marshalizer, mapTx return txsSize } -func computeSizeOfMapTxs(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandler) int { - txsSize := 0 - for _, tx := range mapTxs { - txsSize += computeTxSize(marshalizer, tx) - } - - return txsSize -} - func computeTxSize(marshalizer marshal.Marshalizer, tx data.TransactionHandler) int { txBytes, err := marshalizer.Marshal(tx) if err != nil { From f6dd4fb19165a1145eca3760c3093b42cc2700f9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 29 Jul 2022 11:29:23 +0300 Subject: [PATCH 21/96] integrated new elrond-go-core (changed name of a package) --- data/account.go | 6 +- dataIndexer.go | 10 +-- dataIndexer_test.go | 8 +- go.mod | 2 +- go.sum | 4 +- .../accountsBalanceNftTransfer_test.go | 1 - .../accountsBalanceWithLowerTimestamp_test.go | 1 - .../accountsESDTWithTokenType_test.go | 1 - integrationtests/esdtTransfer_test.go | 1 - .../issueTokenAndSetRoles_test.go | 1 - integrationtests/issueToken_test.go | 1 - integrationtests/nftIssueCreateBurn_test.go | 1 - .../nftTransferCrossShard_test.go | 1 - .../nftTransferCrossWithScCall_test.go | 1 - integrationtests/relayedTx_test.go | 1 - integrationtests/scCallIntraShard_test.go | 1 - integrationtests/transactions_test.go | 1 - integrationtests/updateNFT_test.go | 1 - interface.go | 14 ++-- mock/dbAccountsHandlerStub.go | 6 +- mock/dbTransactionsHandlerStub.go | 6 +- mock/elasticProcessorStub.go | 10 +-- nilIndexer.go | 10 +-- process/accounts/accountsProcessor.go | 10 +-- process/accounts/accountsProcessor_test.go | 44 +++++----- process/block/blockProcessor.go | 4 +- process/block/blockProcessor_test.go | 14 ++-- process/elasticProcessor.go | 12 +-- process/elasticProcessor_test.go | 20 ++--- process/interface.go | 10 +-- process/transactions/checkers.go | 4 +- process/transactions/checkers_test.go | 14 ++-- .../transactions/transactionsGrouper_test.go | 18 ++-- process/transactions/transactionsProcessor.go | 4 +- .../transactionsProcessor_test.go | 82 +++++++++---------- workItems/interface.go | 6 +- workItems/workItemAccounts.go | 6 +- workItems/workItemAccounts_test.go | 6 +- workItems/workItemBlock.go | 10 +-- workItems/workItemBlock_test.go | 42 +++++----- 40 files changed, 191 insertions(+), 204 deletions(-) diff --git a/data/account.go b/data/account.go index 4dd08ecf..9a720134 100644 --- a/data/account.go +++ b/data/account.go @@ -3,7 +3,7 @@ package data import ( "time" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) // AccountInfo holds (serializable) data about an account @@ -55,13 +55,13 @@ type AccountBalanceHistory struct { // Account is a structure that is needed for regular accounts type Account struct { - UserAccount *indexer.AlteredAccount + UserAccount *outport.AlteredAccount IsSender bool } // AccountESDT is a structure that is needed for ESDT accounts type AccountESDT struct { - Account *indexer.AlteredAccount + Account *outport.AlteredAccount TokenIdentifier string NFTNonce uint64 IsSender bool diff --git a/dataIndexer.go b/dataIndexer.go index d6a78a1b..97aac835 100644 --- a/dataIndexer.go +++ b/dataIndexer.go @@ -5,7 +5,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/workItems" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/marshal" ) @@ -59,7 +59,7 @@ func checkIndexerArgs(arguments ArgDataIndexer) error { } // SaveBlock saves the block info in the queue to be sent to elastic -func (di *dataIndexer) SaveBlock(args *indexer.ArgsSaveBlockData) error { +func (di *dataIndexer) SaveBlock(args *outport.ArgsSaveBlockData) error { wi := workItems.NewItemBlock( di.elasticProcessor, di.marshalizer, @@ -88,7 +88,7 @@ func (di *dataIndexer) RevertIndexedBlock(header coreData.HeaderHandler, body co } // SaveRoundsInfo will save data about a slice of rounds in elasticsearch -func (di *dataIndexer) SaveRoundsInfo(rf []*indexer.RoundInfo) error { +func (di *dataIndexer) SaveRoundsInfo(rf []*outport.RoundInfo) error { roundsInfo := make([]*data.RoundInfo, 0) for _, info := range rf { roundsInfo = append(roundsInfo, &data.RoundInfo{ @@ -108,7 +108,7 @@ func (di *dataIndexer) SaveRoundsInfo(rf []*indexer.RoundInfo) error { } // SaveValidatorsRating will save all validators rating info to elasticsearch -func (di *dataIndexer) SaveValidatorsRating(indexID string, validatorsRatingInfo []*indexer.ValidatorRatingInfo) error { +func (di *dataIndexer) SaveValidatorsRating(indexID string, validatorsRatingInfo []*outport.ValidatorRatingInfo) error { valRatingInfo := make([]*data.ValidatorRatingInfo, 0) for _, info := range validatorsRatingInfo { valRatingInfo = append(valRatingInfo, &data.ValidatorRatingInfo{ @@ -140,7 +140,7 @@ func (di *dataIndexer) SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]by } // SaveAccounts will save the provided accounts -func (di *dataIndexer) SaveAccounts(timestamp uint64, accounts map[string]*indexer.AlteredAccount) error { +func (di *dataIndexer) SaveAccounts(timestamp uint64, accounts map[string]*outport.AlteredAccount) error { wi := workItems.NewItemAccounts(di.elasticProcessor, timestamp, accounts) di.dispatcher.Add(wi) diff --git a/dataIndexer_test.go b/dataIndexer_test.go index 39d7be93..99105f30 100644 --- a/dataIndexer_test.go +++ b/dataIndexer_test.go @@ -8,7 +8,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/stretchr/testify/require" ) @@ -69,7 +69,7 @@ func TestDataIndexer_SaveBlock(t *testing.T) { } ei, _ := NewDataIndexer(arguments) - args := &indexer.ArgsSaveBlockData{ + args := &outport.ArgsSaveBlockData{ HeaderHash: []byte("hash"), Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{}}, Header: nil, @@ -96,7 +96,7 @@ func TestDataIndexer_SaveRoundInfo(t *testing.T) { ei, _ := NewDataIndexer(arguments) _ = ei.Close() - err := ei.SaveRoundsInfo([]*indexer.RoundInfo{}) + err := ei.SaveRoundsInfo([]*outport.RoundInfo{}) require.True(t, called) require.Nil(t, err) } @@ -134,7 +134,7 @@ func TestDataIndexer_SaveValidatorsRating(t *testing.T) { } ei, _ := NewDataIndexer(arguments) - err := ei.SaveValidatorsRating("ID", []*indexer.ValidatorRatingInfo{ + err := ei.SaveValidatorsRating("ID", []*outport.ValidatorRatingInfo{ {Rating: 1}, {Rating: 2}, }) require.True(t, called) diff --git a/go.mod b/go.mod index b276231e..5724d51b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727124042-48330ce8d452 + github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220729074346-fca5b87ec7c7 github.com/ElrondNetwork/elrond-go-logger v1.0.7 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index e9e9af55..bffa67b7 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727124042-48330ce8d452 h1:xYVzM0FOxJrCOl8kuHT0Jy6ZfjEJakawVSCqE11QgDc= -github.com/ElrondNetwork/elrond-go-core v1.1.16-0.20220727124042-48330ce8d452/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220729074346-fca5b87ec7c7 h1:31AlKjtaTkdld6Nc28g/hBgV386ft7GbxWwi7Vp0evY= +github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220729074346-fca5b87ec7c7/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7 h1:Ldl1rVS0RGKc1IsW8jIaGCb6Zwei04gsMvyjL05X6mE= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 78c6c442..12be3645 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -12,7 +12,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 5efc3805..9d723158 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -13,7 +13,6 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index a31d84c3..e2f11518 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -15,7 +15,6 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 016a4633..518356f3 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 9bbcd176..ab573de0 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 99f12ed8..f20f27ce 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index d0b6cfea..eb8bef59 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -13,7 +13,6 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index fd5ae3c7..f95182c1 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 19ade6c3..e23da30c 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 52407822..a006f48d 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 0aaafcc5..052e7bc6 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index ddac43e9..ec1ff15a 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index 793ecdaa..acaba366 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -13,7 +13,6 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/interface.go b/interface.go index d3c2d227..28ffc8a5 100644 --- a/interface.go +++ b/interface.go @@ -7,7 +7,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/workItems" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) // DispatcherHandler defines the interface for the dispatcher that will manage when items are saved in elasticsearch database @@ -25,14 +25,14 @@ type ElasticProcessor interface { signersIndexes []uint64, body *block.Body, notarizedHeadersHashes []string, - gasConsumptionData indexer.HeaderGasConsumption, + gasConsumptionData outport.HeaderGasConsumption, txsSize int, ) error RemoveHeader(header coreData.HeaderHandler) error RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error SaveRoundsInfo(infos []*data.RoundInfo) error SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error @@ -61,12 +61,12 @@ type ShardCoordinator interface { // Indexer is an interface for saving node specific data to other storage. // This could be an elastic search index, a MySql database or any other external services. type Indexer interface { - SaveBlock(args *indexer.ArgsSaveBlockData) error + SaveBlock(args *outport.ArgsSaveBlockData) error RevertIndexedBlock(header coreData.HeaderHandler, body coreData.BodyHandler) error - SaveRoundsInfo(roundsInfos []*indexer.RoundInfo) error + SaveRoundsInfo(roundsInfos []*outport.RoundInfo) error SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error - SaveValidatorsRating(indexID string, infoRating []*indexer.ValidatorRatingInfo) error - SaveAccounts(blockTimestamp uint64, acc map[string]*indexer.AlteredAccount) error + SaveValidatorsRating(indexID string, infoRating []*outport.ValidatorRatingInfo) error + SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount) error FinalizedBlock(headerHash []byte) error Close() error IsInterfaceNil() bool diff --git a/mock/dbAccountsHandlerStub.go b/mock/dbAccountsHandlerStub.go index 1cefbc0b..f0c24068 100644 --- a/mock/dbAccountsHandlerStub.go +++ b/mock/dbAccountsHandlerStub.go @@ -2,7 +2,7 @@ package mock import ( "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) // DBAccountsHandlerStub - @@ -12,7 +12,7 @@ type DBAccountsHandlerStub struct { } // GetAccounts - -func (dba *DBAccountsHandlerStub) GetAccounts(_ data.AlteredAccountsHandler, _ map[string]*indexer.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { +func (dba *DBAccountsHandlerStub) GetAccounts(_ data.AlteredAccountsHandler, _ map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { return nil, nil } @@ -59,7 +59,7 @@ func (dba *DBAccountsHandlerStub) SerializeNFTCreateInfo(_ []*data.TokenInfo, _ } // PutTokenMedataDataInTokens - -func (dba *DBAccountsHandlerStub) PutTokenMedataDataInTokens(_ []*data.TokenInfo, _ map[string]*indexer.AlteredAccount) { +func (dba *DBAccountsHandlerStub) PutTokenMedataDataInTokens(_ []*data.TokenInfo, _ map[string]*outport.AlteredAccount) { } // SerializeTypeForProvidedIDs - diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index 4f9706fc..b9151c96 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -4,12 +4,12 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) // DBTransactionProcessorStub - type DBTransactionProcessorStub struct { - PrepareTransactionsForDatabaseCalled func(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) *data.PreparedResults + PrepareTransactionsForDatabaseCalled func(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) *data.PreparedResults SerializeReceiptsCalled func(recs []*data.Receipt, buffSlice *data.BufferSlice, index string) error SerializeScResultsCalled func(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } @@ -19,7 +19,7 @@ func (tps *DBTransactionProcessorStub) SerializeTransactionWithRefund(_ map[stri } // PrepareTransactionsForDatabase - -func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) *data.PreparedResults { +func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) *data.PreparedResults { if tps.PrepareTransactionsForDatabaseCalled != nil { return tps.PrepareTransactionsForDatabaseCalled(body, header, pool) } diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index 74edba1b..8bf08fb3 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -4,7 +4,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) // ElasticProcessorStub - @@ -14,14 +14,14 @@ type ElasticProcessorStub struct { signersIndexes []uint64, body *block.Body, notarizedHeadersHashes []string, - gasConsumptionData indexer.HeaderGasConsumption, + gasConsumptionData outport.HeaderGasConsumption, txsSize int, ) error RemoveHeaderCalled func(header coreData.HeaderHandler) error RemoveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error RemoveTransactionsCalled func(header coreData.HeaderHandler, body *block.Body) error SaveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error - SaveTransactionsCalled func(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error + SaveTransactionsCalled func(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error SaveValidatorsRatingCalled func(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error SaveRoundsInfoCalled func(infos []*data.RoundInfo) error SaveShardValidatorsPubKeysCalled func(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error @@ -34,7 +34,7 @@ func (eim *ElasticProcessorStub) SaveHeader( signersIndexes []uint64, body *block.Body, notarizedHeadersHashes []string, - gasConsumptionData indexer.HeaderGasConsumption, + gasConsumptionData outport.HeaderGasConsumption, txsSize int) error { if eim.SaveHeaderCalled != nil { return eim.SaveHeaderCalled(header, signersIndexes, body, notarizedHeadersHashes, gasConsumptionData, txsSize) @@ -75,7 +75,7 @@ func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, b } // SaveTransactions - -func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { +func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { if eim.SaveTransactionsCalled != nil { return eim.SaveTransactionsCalled(body, header, pool, coreAlteredAccounts) } diff --git a/nilIndexer.go b/nilIndexer.go index afb6ee36..000127f2 100644 --- a/nilIndexer.go +++ b/nilIndexer.go @@ -2,7 +2,7 @@ package indexer import ( "github.com/ElrondNetwork/elrond-go-core/data" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) // NilIndexer will be used when an Indexer is required, but another one isn't necessary or available @@ -15,7 +15,7 @@ func NewNilIndexer() *NilIndexer { } // SaveBlock returns nil -func (ni *NilIndexer) SaveBlock(_ *indexer.ArgsSaveBlockData) error { +func (ni *NilIndexer) SaveBlock(_ *outport.ArgsSaveBlockData) error { return nil } @@ -25,12 +25,12 @@ func (ni *NilIndexer) RevertIndexedBlock(_ data.HeaderHandler, _ data.BodyHandle } // SaveRoundsInfo returns nil -func (ni *NilIndexer) SaveRoundsInfo(_ []*indexer.RoundInfo) error { +func (ni *NilIndexer) SaveRoundsInfo(_ []*outport.RoundInfo) error { return nil } // SaveValidatorsRating returns nil -func (ni *NilIndexer) SaveValidatorsRating(_ string, _ []*indexer.ValidatorRatingInfo) error { +func (ni *NilIndexer) SaveValidatorsRating(_ string, _ []*outport.ValidatorRatingInfo) error { return nil } @@ -40,7 +40,7 @@ func (ni *NilIndexer) SaveValidatorsPubKeys(_ map[uint32][][]byte, _ uint32) err } // SaveAccounts returns nil -func (ni *NilIndexer) SaveAccounts(_ uint64, _ map[string]*indexer.AlteredAccount) error { +func (ni *NilIndexer) SaveAccounts(_ uint64, _ map[string]*outport.AlteredAccount) error { return nil } diff --git a/process/accounts/accountsProcessor.go b/process/accounts/accountsProcessor.go index 543267c6..c5135bd0 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/accounts/accountsProcessor.go @@ -12,7 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" logger "github.com/ElrondNetwork/elrond-go-logger" ) @@ -44,7 +44,7 @@ func NewAccountsProcessor( // TODO: refactor this as the altered accounts are already computed on the node. EN-12389 // GetAccounts will get accounts for regular operations and esdt operations -func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { +func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) @@ -70,7 +70,7 @@ func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHan } func splitAlteredAccounts( - account *coreIndexerData.AlteredAccount, + account *outport.AlteredAccount, altered []*data.AlteredAccount, ) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) @@ -257,7 +257,7 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In } // PutTokenMedataDataInTokens will put the TokenMedata in provided tokens data -func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount) { +func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*outport.AlteredAccount) { for _, tokenData := range tokensData { if tokenData.Data != nil || tokenData.Nonce == 0 { continue @@ -278,7 +278,7 @@ func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.Token func (ap *accountsProcessor) loadMetadataForToken( tokenData *data.TokenInfo, - coreAlteredAccounts map[string]*coreIndexerData.AlteredAccount, + coreAlteredAccounts map[string]*outport.AlteredAccount, ) (*esdt.MetaData, error) { for _, account := range coreAlteredAccounts { for _, token := range account.Tokens { diff --git a/process/accounts/accountsProcessor_test.go b/process/accounts/accountsProcessor_test.go index 31af1e6b..8892ca8e 100644 --- a/process/accounts/accountsProcessor_test.go +++ b/process/accounts/accountsProcessor_test.go @@ -15,7 +15,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/process/tags" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/stretchr/testify/require" ) @@ -87,11 +87,11 @@ func TestGetESDTInfo(t *testing.T) { tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &coreIndexerData.AlteredAccount{ + Account: &outport.AlteredAccount{ Address: "", Balance: "1000", Nonce: 0, - Tokens: []*coreIndexerData.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: tokenIdentifier, Balance: "1000", @@ -115,11 +115,11 @@ func TestGetESDTInfoNFT(t *testing.T) { tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &coreIndexerData.AlteredAccount{ + Account: &outport.AlteredAccount{ Address: "", Balance: "1", Nonce: 10, - Tokens: []*coreIndexerData.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: tokenIdentifier, Balance: "1", @@ -150,11 +150,11 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &coreIndexerData.AlteredAccount{ + Account: &outport.AlteredAccount{ Address: "", Balance: "1", Nonce: 1, - Tokens: []*coreIndexerData.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: tokenIdentifier, Balance: "1", @@ -188,8 +188,8 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { t.Parallel() addr := "aaaabbbb" - acc := &coreIndexerData.AlteredAccount{} - alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ + acc := &outport.AlteredAccount{} + alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) @@ -214,11 +214,11 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { t.Parallel() addr := "aaaabbbb" - acc := &coreIndexerData.AlteredAccount{ + acc := &outport.AlteredAccount{ Address: addr, Balance: "1", } - alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ + alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) @@ -240,8 +240,8 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco t.Parallel() addr := "aaaabbbb" - acc := &coreIndexerData.AlteredAccount{} - alteredAccountsMap := map[string]*coreIndexerData.AlteredAccount{ + acc := &outport.AlteredAccount{} + alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) @@ -269,7 +269,7 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { addrBytes := bytes.Repeat([]byte{0}, 32) addr := hex.EncodeToString(addrBytes) - account := &coreIndexerData.AlteredAccount{ + account := &outport.AlteredAccount{ Address: addr, Balance: "1000", Nonce: 1, @@ -292,7 +292,7 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { TotalBalanceWithStake: "1000", TotalBalanceWithStakeNum: balanceConverter.ComputeBalanceAsFloat(big.NewInt(1000)), IsSmartContract: true, - Timestamp: time.Duration(123), + Timestamp: time.Duration(123), }, res[addr]) } @@ -302,9 +302,9 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { addr := "aaaabbbb" - account := &coreIndexerData.AlteredAccount{ + account := &outport.AlteredAccount{ Address: hex.EncodeToString([]byte(addr)), - Tokens: []*coreIndexerData.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Balance: "1000", Identifier: "token", @@ -406,7 +406,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { tokensInfo := []*data.TokenInfo{ {Data: nil}, {Nonce: 5, Data: &data.TokenMetaData{Creator: oldCreator}}, } - emptyAlteredAccounts := map[string]*coreIndexerData.AlteredAccount{} + emptyAlteredAccounts := map[string]*outport.AlteredAccount{} ap.PutTokenMedataDataInTokens(tokensInfo, emptyAlteredAccounts) require.Empty(t, emptyAlteredAccounts) require.Empty(t, tokensInfo[0].Data) @@ -426,8 +426,8 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { }, } - alteredAccounts := map[string]*coreIndexerData.AlteredAccount{ - "addr": {Tokens: []*coreIndexerData.AccountTokenData{}}, + alteredAccounts := map[string]*outport.AlteredAccount{ + "addr": {Tokens: []*outport.AccountTokenData{}}, } ap.PutTokenMedataDataInTokens(tokensInfo, alteredAccounts) require.Empty(t, tokensInfo[0].Data) @@ -452,9 +452,9 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { }, } - alteredAccounts := map[string]*coreIndexerData.AlteredAccount{ + alteredAccounts := map[string]*outport.AlteredAccount{ "addr0": { - Tokens: []*coreIndexerData.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "token0-5t6y7u", Nonce: 5, diff --git a/process/block/blockProcessor.go b/process/block/blockProcessor.go index b7464073..d38f8893 100644 --- a/process/block/blockProcessor.go +++ b/process/block/blockProcessor.go @@ -14,7 +14,7 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" nodeBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" @@ -48,7 +48,7 @@ func (bp *blockProcessor) PrepareBlockForDB( signersIndexes []uint64, body *block.Body, notarizedHeadersHashes []string, - gasConsumptionData coreIndexerData.HeaderGasConsumption, + gasConsumptionData outport.HeaderGasConsumption, sizeTxs int, ) (*data.Block, error) { if check.IfNil(header) { diff --git a/process/block/blockProcessor_test.go b/process/block/blockProcessor_test.go index 2e93b07a..614151ba 100644 --- a/process/block/blockProcessor_test.go +++ b/process/block/blockProcessor_test.go @@ -11,7 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elrond-go-core/core" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - coreIndexerData "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/stretchr/testify/require" @@ -71,7 +71,7 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { ReceiverShardID: 2, }, }, - }, nil, coreIndexerData.HeaderGasConsumption{}, 0) + }, nil, outport.HeaderGasConsumption{}, 0) require.Nil(t, err) expectedBlock := &data.Block{ @@ -93,7 +93,7 @@ func TestBlockProcessor_PrepareBlockForDBNilHeader(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB(nil, nil, &dataBlock.Body{}, nil, coreIndexerData.HeaderGasConsumption{}, 0) + dbBlock, err := bp.PrepareBlockForDB(nil, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0) require.Equal(t, indexer.ErrNilHeaderHandler, err) require.Nil(t, dbBlock) } @@ -103,7 +103,7 @@ func TestBlockProcessor_PrepareBlockForDBNilBody(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB(&dataBlock.MetaBlock{}, nil, nil, nil, coreIndexerData.HeaderGasConsumption{}, 0) + dbBlock, err := bp.PrepareBlockForDB(&dataBlock.MetaBlock{}, nil, nil, nil, outport.HeaderGasConsumption{}, 0) require.Equal(t, indexer.ErrNilBlockBody, err) require.Nil(t, dbBlock) } @@ -118,7 +118,7 @@ func TestBlockProcessor_PrepareBlockForDBMarshalFailHeader(t *testing.T) { }, }) - dbBlock, err := bp.PrepareBlockForDB(&dataBlock.MetaBlock{}, nil, &dataBlock.Body{}, nil, coreIndexerData.HeaderGasConsumption{}, 0) + dbBlock, err := bp.PrepareBlockForDB(&dataBlock.MetaBlock{}, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0) require.Equal(t, expectedErr, err) require.Nil(t, dbBlock) } @@ -140,7 +140,7 @@ func TestBlockProcessor_PrepareBlockForDBMarshalFailBlock(t *testing.T) { }, }) - dbBlock, err := bp.PrepareBlockForDB(&dataBlock.MetaBlock{}, nil, &dataBlock.Body{}, nil, coreIndexerData.HeaderGasConsumption{}, 0) + dbBlock, err := bp.PrepareBlockForDB(&dataBlock.MetaBlock{}, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0) require.Equal(t, expectedErr, err) require.Nil(t, dbBlock) } @@ -184,7 +184,7 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { TxCount: 120, }, }, - }, nil, &dataBlock.Body{}, nil, coreIndexerData.HeaderGasConsumption{}, 0) + }, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0) require.Equal(t, nil, err) require.Equal(t, &data.Block{ Nonce: 0, diff --git a/process/elasticProcessor.go b/process/elasticProcessor.go index b0957b19..eb3f7fdf 100644 --- a/process/elasticProcessor.go +++ b/process/elasticProcessor.go @@ -14,7 +14,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/elastic/go-elasticsearch/v7/esapi" ) @@ -251,7 +251,7 @@ func (ei *elasticProcessor) SaveHeader( signersIndexes []uint64, body *block.Body, notarizedHeadersHashes []string, - gasConsumptionData indexer.HeaderGasConsumption, + gasConsumptionData outport.HeaderGasConsumption, txsSize int, ) error { if !ei.isIndexEnabled(elasticIndexer.BlockIndex) { @@ -351,8 +351,8 @@ func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock) (map[string func (ei *elasticProcessor) SaveTransactions( body *block.Body, header coreData.HeaderHandler, - pool *indexer.Pool, - coreAlteredAccounts map[string]*indexer.AlteredAccount, + pool *outport.Pool, + coreAlteredAccounts map[string]*outport.AlteredAccount, ) error { headerTimestamp := header.GetTimeStamp() @@ -575,7 +575,7 @@ func (ei *elasticProcessor) indexAlteredAccounts( timestamp uint64, alteredAccounts data.AlteredAccountsHandler, updatesNFTsData []*data.NFTDataUpdate, - coreAlteredAccounts map[string]*indexer.AlteredAccount, + coreAlteredAccounts map[string]*outport.AlteredAccount, buffSlice *data.BufferSlice, tagsCount data.CountTags, ) error { @@ -653,7 +653,7 @@ func (ei *elasticProcessor) indexAccountsESDT( return ei.accountsProc.SerializeAccountsESDT(accountsESDTMap, updatesNFTsData, buffSlice, elasticIndexer.AccountsESDTIndex) } -func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, coreAlteredAccounts map[string]*indexer.AlteredAccount, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, coreAlteredAccounts map[string]*outport.AlteredAccount, buffSlice *data.BufferSlice) error { shouldSkipIndex := !ei.isIndexEnabled(elasticIndexer.TokensIndex) || tokensData.Len() == 0 if shouldSkipIndex { return nil diff --git a/process/elasticProcessor_test.go b/process/elasticProcessor_test.go index f67c0b2e..4ddd3008 100644 --- a/process/elasticProcessor_test.go +++ b/process/elasticProcessor_test.go @@ -27,7 +27,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/stretchr/testify/require" @@ -83,7 +83,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { func newTestTxPool() map[string]coreData.TransactionHandlerWithGasUsedAndFee { txPool := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "tx1": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + "tx1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: uint64(1), Value: big.NewInt(1), RcvAddr: []byte("receiver_address1"), @@ -93,7 +93,7 @@ func newTestTxPool() map[string]coreData.TransactionHandlerWithGasUsedAndFee { Data: []byte("tx_data1"), Signature: []byte("signature1"), }, 0, big.NewInt(0)), - "tx2": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + "tx2": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: uint64(2), Value: big.NewInt(2), RcvAddr: []byte("receiver_address2"), @@ -103,7 +103,7 @@ func newTestTxPool() map[string]coreData.TransactionHandlerWithGasUsedAndFee { Data: []byte("tx_data2"), Signature: []byte("signature2"), }, 0, big.NewInt(0)), - "tx3": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + "tx3": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: uint64(3), Value: big.NewInt(3), RcvAddr: []byte("receiver_address3"), @@ -360,7 +360,7 @@ func TestElasticseachDatabaseSaveHeader_RequestError(t *testing.T) { arguments.BlockProc, _ = block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveHeader(header, signerIndexes, &dataBlock.Body{}, nil, indexer.HeaderGasConsumption{}, 1) + err := elasticDatabase.SaveHeader(header, signerIndexes, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 1) require.Equal(t, localErr, err) } @@ -401,7 +401,7 @@ func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { arguments.BlockProc, _ = block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveHeader(header, signerIndexes, blockBody, nil, indexer.HeaderGasConsumption{}, 1) + err := elasticDatabase.SaveHeader(header, signerIndexes, blockBody, nil, outport.HeaderGasConsumption{}, 1) require.Nil(t, err) } @@ -430,7 +430,7 @@ func TestElasticseachSaveTransactions(t *testing.T) { arguments.TransactionsProc = txDbProc elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - pool := &indexer.Pool{Txs: txPool} + pool := &outport.Pool{Txs: txPool} err := elasticDatabase.SaveTransactions(body, header, pool, nil) require.Equal(t, localErr, err) } @@ -626,7 +626,7 @@ func TestElasticProcessor_IndexEpochInfoData(t *testing.T) { body := &dataBlock.Body{} metaHeader := &dataBlock.MetaBlock{} - err = elasticSearchProc.SaveHeader(metaHeader, nil, body, nil, indexer.HeaderGasConsumption{}, 0) + err = elasticSearchProc.SaveHeader(metaHeader, nil, body, nil, outport.HeaderGasConsumption{}, 0) require.Nil(t, err) require.True(t, called) } @@ -635,7 +635,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) called := false arguments := createMockElasticProcessorArgs() arguments.TransactionsProc = &mock.DBTransactionProcessorStub{ - PrepareTransactionsForDatabaseCalled: func(body *dataBlock.Body, header coreData.HeaderHandler, pool *indexer.Pool) *data.PreparedResults { + PrepareTransactionsForDatabaseCalled: func(body *dataBlock.Body, header coreData.HeaderHandler, pool *outport.Pool) *data.PreparedResults { return &data.PreparedResults{ Transactions: nil, ScResults: nil, @@ -657,7 +657,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.enabledIndexes[elasticIndexer.ScResultsIndex] = struct{}{} - err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &indexer.Pool{}, nil) + err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &outport.Pool{}, nil) require.Nil(t, err) require.False(t, called) } diff --git a/process/interface.go b/process/interface.go index ce5a6b5e..d67edb00 100644 --- a/process/interface.go +++ b/process/interface.go @@ -7,7 +7,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/elastic/go-elasticsearch/v7/esapi" ) @@ -30,11 +30,11 @@ type DatabaseClientHandler interface { // DBAccountHandler defines the actions that an accounts' handler should do type DBAccountHandler interface { - GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*indexer.AlteredAccount) ([]*data.Account, []*data.AccountESDT) + GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) PrepareRegularAccountsMap(timestamp uint64, accounts []*data.Account) map[string]*data.AccountInfo PrepareAccountsMapESDT(timestamp uint64, accounts []*data.AccountESDT, tagsCount data.CountTags) (map[string]*data.AccountInfo, data.TokensHandler) PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo) map[string]*data.AccountBalanceHistory - PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*indexer.AlteredAccount) + PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*outport.AlteredAccount) SerializeAccountsHistory(accounts map[string]*data.AccountBalanceHistory, buffSlice *data.BufferSlice, index string) error SerializeAccounts(accounts map[string]*data.AccountInfo, buffSlice *data.BufferSlice, index string) error @@ -50,7 +50,7 @@ type DBBlockHandler interface { signersIndexes []uint64, body *block.Body, notarizedHeadersHashes []string, - gasConsumptionData indexer.HeaderGasConsumption, + gasConsumptionData outport.HeaderGasConsumption, sizeTxs int, ) (*data.Block, error) ComputeHeaderHash(header coreData.HeaderHandler) ([]byte, error) @@ -64,7 +64,7 @@ type DBTransactionsHandler interface { PrepareTransactionsForDatabase( body *block.Body, header coreData.HeaderHandler, - pool *indexer.Pool, + pool *outport.Pool, ) *data.PreparedResults GetRewardsTxsHashesHexEncoded(header coreData.HeaderHandler, body *block.Body) []string diff --git a/process/transactions/checkers.go b/process/transactions/checkers.go index ae7d9bd1..c3329752 100644 --- a/process/transactions/checkers.go +++ b/process/transactions/checkers.go @@ -12,7 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" vmcommon "github.com/ElrondNetwork/elrond-vm-common" ) @@ -46,7 +46,7 @@ func checkTxsProcessorArg(args *ArgsTransactionProcessor) error { func checkPrepareTransactionForDatabaseArguments( body *block.Body, header coreData.HeaderHandler, - pool *indexer.Pool, + pool *outport.Pool, ) error { if body == nil { return elasticIndexer.ErrNilBlockBody diff --git a/process/transactions/checkers_test.go b/process/transactions/checkers_test.go index 6d80f9c9..5c477c93 100644 --- a/process/transactions/checkers_test.go +++ b/process/transactions/checkers_test.go @@ -11,7 +11,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) @@ -93,26 +93,26 @@ func TestCheckTxsProcessorArg(t *testing.T) { tests := []struct { name string - args func() (body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) + args func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) exErr error }{ { name: "NilBlockBody", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) { - return nil, &block.Header{}, &indexer.Pool{} + args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) { + return nil, &block.Header{}, &outport.Pool{} }, exErr: elasticIndexer.ErrNilBlockBody, }, { name: "NilHeaderHandler", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) { - return &block.Body{}, nil, &indexer.Pool{} + args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) { + return &block.Body{}, nil, &outport.Pool{} }, exErr: elasticIndexer.ErrNilHeaderHandler, }, { name: "NilPool", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool) { + args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) { return &block.Body{}, &block.Header{}, nil }, exErr: elasticIndexer.ErrNilPool, diff --git a/process/transactions/transactionsGrouper_test.go b/process/transactions/transactionsGrouper_test.go index f2346b36..975de455 100644 --- a/process/transactions/transactionsGrouper_test.go +++ b/process/transactions/transactionsGrouper_test.go @@ -8,7 +8,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/receipt" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" "github.com/ElrondNetwork/elrond-go-core/data/transaction" @@ -30,11 +30,11 @@ func TestGroupNormalTxs(t *testing.T) { } header := &block.Header{} txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender1"), RcvAddr: []byte("receiver1"), }, 0, big.NewInt(0)), - string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender2"), RcvAddr: []byte("receiver2"), }, 0, big.NewInt(0)), @@ -61,10 +61,10 @@ func TestGroupRewardsTxs(t *testing.T) { } header := &block.Header{} txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ + string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ RcvAddr: []byte("receiver1"), }, 0, big.NewInt(0)), - string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ + string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ RcvAddr: []byte("receiver2"), }, 0, big.NewInt(0)), } @@ -90,11 +90,11 @@ func TestGroupInvalidTxs(t *testing.T) { } header := &block.Header{} txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender1"), RcvAddr: []byte("receiver1"), }, 0, big.NewInt(0)), - string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("sender2"), RcvAddr: []byte("receiver2"), }, 0, big.NewInt(0)), @@ -117,10 +117,10 @@ func TestGroupReceipts(t *testing.T) { txHash2 := []byte("txHash2") header := &block.Header{} txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ SndAddr: []byte("sender1"), }, 0, big.NewInt(0)), - string(txHash2): indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ SndAddr: []byte("sender2"), }, 0, big.NewInt(0)), } diff --git a/process/transactions/transactionsProcessor.go b/process/transactions/transactionsProcessor.go index e3cb786a..cda3504c 100644 --- a/process/transactions/transactionsProcessor.go +++ b/process/transactions/transactionsProcessor.go @@ -9,7 +9,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - indexerArgs "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" @@ -78,7 +78,7 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( body *block.Body, header coreData.HeaderHandler, - pool *indexerArgs.Pool, + pool *outport.Pool, ) *data.PreparedResults { err := checkPrepareTransactionForDatabaseArguments(body, header, pool) if err != nil { diff --git a/process/transactions/transactionsProcessor_test.go b/process/transactions/transactionsProcessor_test.go index db9836f7..c213d82a 100644 --- a/process/transactions/transactionsProcessor_test.go +++ b/process/transactions/transactionsProcessor_test.go @@ -3,6 +3,7 @@ package transactions import ( "encoding/hex" "fmt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -12,7 +13,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/pubkeyConverter" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" "github.com/ElrondNetwork/elrond-go-core/data/receipt" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" @@ -92,52 +92,52 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { t.Parallel() txHash1 := []byte("txHash1") - tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, }, 0, big.NewInt(0)) txHash2 := []byte("txHash2") - tx2 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx2 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, }, 0, big.NewInt(0)) txHash3 := []byte("txHash3") - tx3 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) + tx3 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) txHash4 := []byte("txHash4") - tx4 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) + tx4 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) txHash5 := []byte("txHash5") - tx5 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) + tx5 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) rTx1Hash := []byte("rTxHash1") - rTx1 := indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) + rTx1 := outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) rTx2Hash := []byte("rTxHash2") - rTx2 := indexer.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) + rTx2 := outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) recHash1 := []byte("recHash1") - rec1 := indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + rec1 := outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ Value: big.NewInt(100), TxHash: txHash1, }, 0, big.NewInt(0)) recHash2 := []byte("recHash2") - rec2 := indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + rec2 := outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ Value: big.NewInt(200), TxHash: txHash2, }, 0, big.NewInt(0)) scHash1 := []byte("scHash1") - scResult1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scResult1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, }, 0, big.NewInt(0)) scHash2 := []byte("scHash2") - scResult2 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scResult2 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, }, 0, big.NewInt(0)) scHash3 := []byte("scHash3") - scResult3 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scResult3 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash3, Data: []byte("@" + "6F6B"), }, 0, big.NewInt(0)) @@ -176,7 +176,7 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { } header := &block.Header{} - pool := &indexer.Pool{ + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, string(txHash2): tx2, @@ -212,20 +212,20 @@ func TestRelayedTransactions(t *testing.T) { t.Parallel() txHash1 := []byte("txHash1") - tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, Data: []byte("relayedTx@blablabllablalba"), }, 0, big.NewInt(0)) scHash1 := []byte("scHash1") - scResult1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scResult1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, }, 0, big.NewInt(0)) scHash2 := []byte("scHash2") - scResult2 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scResult2 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, @@ -246,7 +246,7 @@ func TestRelayedTransactions(t *testing.T) { header := &block.Header{} - pool := &indexer.Pool{ + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, @@ -300,7 +300,7 @@ func TestAlteredAddresses(t *testing.T) { address1 := []byte("address1") // should be added address2 := []byte("address2") expectedAlteredAccounts[hex.EncodeToString(address1)] = struct{}{} - tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: address1, RcvAddr: address2, }, 0, big.NewInt(0)) @@ -309,7 +309,7 @@ func TestAlteredAddresses(t *testing.T) { address3 := []byte("address3") address4 := []byte("address4") // should be added expectedAlteredAccounts[hex.EncodeToString(address4)] = struct{}{} - tx2 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx2 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: address3, RcvAddr: address4, }, 0, big.NewInt(0)) @@ -359,7 +359,7 @@ func TestAlteredAddresses(t *testing.T) { address7 := []byte("address7") // should be added address8 := []byte("address8") expectedAlteredAccounts[hex.EncodeToString(address7)] = struct{}{} - scr1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scr1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ RcvAddr: address7, SndAddr: address8, }, 0, big.NewInt(0)) @@ -368,7 +368,7 @@ func TestAlteredAddresses(t *testing.T) { address9 := []byte("address9") // should be added address10 := []byte("address10") expectedAlteredAccounts[hex.EncodeToString(address9)] = struct{}{} - scr2 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scr2 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ RcvAddr: address9, SndAddr: address10, }, 0, big.NewInt(0)) @@ -387,7 +387,7 @@ func TestAlteredAddresses(t *testing.T) { hdr := &block.Header{} - pool := &indexer.Pool{ + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(tx1Hash): tx1, string(tx2Hash): tx2, @@ -397,8 +397,8 @@ func TestAlteredAddresses(t *testing.T) { string(scr2Hash): scr2, }, Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(rwdTx1Hash): indexer.NewTransactionHandlerWithGasAndFee(rwdTx1, 0, big.NewInt(0)), - string(rwdTx2Hash): indexer.NewTransactionHandlerWithGasAndFee(rwdTx2, 0, big.NewInt(0)), + string(rwdTx1Hash): outport.NewTransactionHandlerWithGasAndFee(rwdTx1, 0, big.NewInt(0)), + string(rwdTx2Hash): outport.NewTransactionHandlerWithGasAndFee(rwdTx2, 0, big.NewInt(0)), }, } @@ -465,12 +465,12 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) txHash1 := []byte("txHash1") - tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, }, 0, big.NewInt(0)) recHash1 := []byte("recHash1") - rec1 := indexer.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + rec1 := outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ Value: big.NewInt(100), TxHash: txHash1, }, 0, big.NewInt(0)) @@ -490,7 +490,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { header := &block.Header{} - pool := &indexer.Pool{ + pool := &outport.Pool{ Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, @@ -554,13 +554,13 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) txHash1 := []byte("txHash1") - tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 123456, Data: []byte("ESDTTransfer@54474e2d383862383366@0a"), }, 0, big.NewInt(0)) scResHash1 := []byte("scResHash1") - scRes1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scRes1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, }, 0, big.NewInt(0)) @@ -579,7 +579,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * header := &block.Header{} - pool := &indexer.Pool{ + pool := &outport.Pool{ Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, @@ -609,13 +609,13 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }} txHash1 := []byte("txHash1") - tx1 := indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 123456, Data: []byte("ESDTNFTTransfer@595959453643392D303837363661@01@01@000000000000000005005C83E0C42EDCE394F40B24D29D298B0249C41F028974@66756E64@890479AFC610F4BEBC087D3ADA3F7C2775C736BBA91F41FD3D65092AA482D8B0@1c20"), }, 0, big.NewInt(0)) scResHash1 := []byte("scResHash1") - scRes1 := indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scRes1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 1, Data: []byte("@" + okHexEncoded), OriginalTxHash: txHash1, @@ -637,7 +637,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t header := &block.Header{} - pool := &indexer.Pool{ + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(txHash1): tx1, }, @@ -687,22 +687,22 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } header := &block.Header{} - pool := &indexer.Pool{ + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "t1": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + "t1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), }, 0, big.NewInt(0)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "scr1": indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + "scr1": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("ESDTTransfer@414141414141412d323436626461@0186a0"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), }, 0, big.NewInt(0)), - "scr2": indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + "scr2": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("@6f6b"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), @@ -719,16 +719,16 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { require.True(t, ok) // transaction fail - pool = &indexer.Pool{ + pool = &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "t1": indexer.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + "t1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), }, 0, big.NewInt(0)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "scr1": indexer.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + "scr1": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("75736572206572726f72"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), diff --git a/workItems/interface.go b/workItems/interface.go index a9f6bb45..57acd283 100644 --- a/workItems/interface.go +++ b/workItems/interface.go @@ -4,7 +4,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) // WorkItemHandler defines the interface for item that needs to be saved in elasticsearch database @@ -19,11 +19,11 @@ type saveBlockIndexer interface { signersIndexes []uint64, body *block.Body, notarizedHeadersHashes []string, - gasConsumptionData indexer.HeaderGasConsumption, + gasConsumptionData outport.HeaderGasConsumption, txsSize int, ) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error } type saveRatingIndexer interface { diff --git a/workItems/workItemAccounts.go b/workItems/workItemAccounts.go index aa7068c3..4ad77fc2 100644 --- a/workItems/workItemAccounts.go +++ b/workItems/workItemAccounts.go @@ -2,20 +2,20 @@ package workItems import ( "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) type itemAccounts struct { indexer saveAccountsIndexer blockTimestamp uint64 - accounts map[string]*indexer.AlteredAccount + accounts map[string]*outport.AlteredAccount } // NewItemAccounts will create a new instance of itemAccounts func NewItemAccounts( indexer saveAccountsIndexer, blockTimestamp uint64, - accounts map[string]*indexer.AlteredAccount, + accounts map[string]*outport.AlteredAccount, ) WorkItemHandler { return &itemAccounts{ indexer: indexer, diff --git a/workItems/workItemAccounts_test.go b/workItems/workItemAccounts_test.go index 341ba235..970d5792 100644 --- a/workItems/workItemAccounts_test.go +++ b/workItems/workItemAccounts_test.go @@ -7,7 +7,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elastic-indexer-go/workItems" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/stretchr/testify/require" ) @@ -21,7 +21,7 @@ func TestItemAccounts_Save(t *testing.T) { }, }, 0, - make(map[string]*indexer.AlteredAccount), + make(map[string]*outport.AlteredAccount), ) require.False(t, itemAccounts.IsInterfaceNil()) @@ -39,7 +39,7 @@ func TestItemAccounts_SaveAccountsShouldErr(t *testing.T) { }, }, 0, - make(map[string]*indexer.AlteredAccount), + make(map[string]*outport.AlteredAccount), ) require.False(t, itemAccounts.IsInterfaceNil()) diff --git a/workItems/workItemBlock.go b/workItems/workItemBlock.go index fa83aa3a..ad6d835f 100644 --- a/workItems/workItemBlock.go +++ b/workItems/workItemBlock.go @@ -9,7 +9,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" ) @@ -22,14 +22,14 @@ var log = logger.GetOrCreate("indexer/workItems") type itemBlock struct { indexer saveBlockIndexer marshalizer marshal.Marshalizer - argsSaveBlock *indexer.ArgsSaveBlockData + argsSaveBlock *outport.ArgsSaveBlockData } // NewItemBlock will create a new instance of ItemBlock func NewItemBlock( indexer saveBlockIndexer, marshalizer marshal.Marshalizer, - args *indexer.ArgsSaveBlockData, + args *outport.ArgsSaveBlockData, ) WorkItemHandler { return &itemBlock{ indexer: indexer, @@ -60,7 +60,7 @@ func (wib *itemBlock) Save() error { } if wib.argsSaveBlock.TransactionsPool == nil { - wib.argsSaveBlock.TransactionsPool = &indexer.Pool{} + wib.argsSaveBlock.TransactionsPool = &outport.Pool{} } txsSizeInBytes := ComputeSizeOfTxs(wib.marshalizer, wib.argsSaveBlock.TransactionsPool) @@ -101,7 +101,7 @@ func (wib *itemBlock) IsInterfaceNil() bool { } // ComputeSizeOfTxs will compute size of transactions in bytes -func ComputeSizeOfTxs(marshalizer marshal.Marshalizer, pool *indexer.Pool) int { +func ComputeSizeOfTxs(marshalizer marshal.Marshalizer, pool *outport.Pool) int { sizeTxs := 0 sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Txs) sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Scrs) diff --git a/workItems/workItemBlock_test.go b/workItems/workItemBlock_test.go index 8df1b532..87118b54 100644 --- a/workItems/workItemBlock_test.go +++ b/workItems/workItemBlock_test.go @@ -12,7 +12,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/workItems" "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/stretchr/testify/assert" @@ -32,7 +32,7 @@ func generateTxs(numTxs int) map[string]data.TransactionHandlerWithGasUsedAndFee Data: []byte("dasjdksakjdksajdjksajkdjkasjdksajkdasjdksakjdksajdjksajkdjkasjdksajkdasjdksakjdksajdjksajkdjkasjdksajk"), Signature: []byte("randomSignatureasdasldkasdsahjgdlhjaskldsjkaldjklasjkdjskladjkl;sajkl"), } - txs[fmt.Sprintf("%d", i)] = indexer.NewTransactionHandlerWithGasAndFee(tx, 0, big.NewInt(0)) + txs[fmt.Sprintf("%d", i)] = outport.NewTransactionHandlerWithGasAndFee(tx, 0, big.NewInt(0)) } return txs @@ -42,7 +42,7 @@ func TestItemBlock_SaveNilHeaderShouldRetNil(t *testing.T) { itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{}, &mock.MarshalizerMock{}, - &indexer.ArgsSaveBlockData{}, + &outport.ArgsSaveBlockData{}, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -54,15 +54,15 @@ func TestItemBlock_SaveHeaderShouldErr(t *testing.T) { localErr := errors.New("local err") itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData indexer.HeaderGasConsumption, txsSize int) error { + SaveHeaderCalled: func(header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData outport.HeaderGasConsumption, txsSize int) error { return localErr }, }, &mock.MarshalizerMock{}, - &indexer.ArgsSaveBlockData{ + &outport.ArgsSaveBlockData{ Header: &dataBlock.Header{}, Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &indexer.Pool{}, + TransactionsPool: &outport.Pool{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -75,7 +75,7 @@ func TestItemBlock_SaveNoMiniblocksShoulCallSaveHeader(t *testing.T) { countCalled := 0 itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData indexer.HeaderGasConsumption, txsSize int) error { + SaveHeaderCalled: func(header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData outport.HeaderGasConsumption, txsSize int) error { countCalled++ return nil }, @@ -83,16 +83,16 @@ func TestItemBlock_SaveNoMiniblocksShoulCallSaveHeader(t *testing.T) { countCalled++ return nil }, - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { + SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { countCalled++ return nil }, }, &mock.MarshalizerMock{}, - &indexer.ArgsSaveBlockData{ + &outport.ArgsSaveBlockData{ Body: &dataBlock.Body{}, Header: &dataBlock.Header{}, - TransactionsPool: &indexer.Pool{}, + TransactionsPool: &outport.Pool{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -111,10 +111,10 @@ func TestItemBlock_SaveMiniblocksShouldErr(t *testing.T) { }, }, &mock.MarshalizerMock{}, - &indexer.ArgsSaveBlockData{ + &outport.ArgsSaveBlockData{ Header: &dataBlock.Header{}, Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &indexer.Pool{}, + TransactionsPool: &outport.Pool{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -127,15 +127,15 @@ func TestItemBlock_SaveTransactionsShouldErr(t *testing.T) { localErr := errors.New("local err") itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { + SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { return localErr }, }, &mock.MarshalizerMock{}, - &indexer.ArgsSaveBlockData{ + &outport.ArgsSaveBlockData{ Header: &dataBlock.Header{}, Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &indexer.Pool{}, + TransactionsPool: &outport.Pool{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -148,7 +148,7 @@ func TestItemBlock_SaveShouldWork(t *testing.T) { countCalled := 0 itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData indexer.HeaderGasConsumption, txsSize int) error { + SaveHeaderCalled: func(header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData outport.HeaderGasConsumption, txsSize int) error { countCalled++ return nil }, @@ -156,16 +156,16 @@ func TestItemBlock_SaveShouldWork(t *testing.T) { countCalled++ return nil }, - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *indexer.Pool, coreAlteredAccounts map[string]*indexer.AlteredAccount) error { + SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { countCalled++ return nil }, }, &mock.MarshalizerMock{}, - &indexer.ArgsSaveBlockData{ + &outport.ArgsSaveBlockData{ Header: &dataBlock.Header{}, Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &indexer.Pool{}, + TransactionsPool: &outport.Pool{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -187,7 +187,7 @@ func benchmarkComputeSizeOfTxsDuration(b *testing.B) { gogoMarsh := &marshal.GogoProtoMarshalizer{} for i := 0; i < b.N; i++ { - workItems.ComputeSizeOfTxs(gogoMarsh, &indexer.Pool{Txs: txs}) + workItems.ComputeSizeOfTxs(gogoMarsh, &outport.Pool{Txs: txs}) } } @@ -197,7 +197,7 @@ func TestComputeSizeOfTxs(t *testing.T) { txs := generateTxs(numTxs) gogoMarsh := &marshal.GogoProtoMarshalizer{} - lenTxs := workItems.ComputeSizeOfTxs(gogoMarsh, &indexer.Pool{Txs: txs}) + lenTxs := workItems.ComputeSizeOfTxs(gogoMarsh, &outport.Pool{Txs: txs}) keys := reflect.ValueOf(txs).MapKeys() oneTxBytes, _ := gogoMarsh.Marshal(txs[keys[0].String()].GetTxHandler()) From c28c768633d43277e594ff3433403ad4006b4955 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 11 Aug 2022 15:25:10 +0300 Subject: [PATCH 22/96] latest elrond go core --- go.mod | 2 +- go.sum | 4 ++-- integrationtests/collectionsIndex_test.go | 20 ++++++++++---------- integrationtests/createNFTWithTags_test.go | 12 ++++++------ 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 5724d51b..de95e3c1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220729074346-fca5b87ec7c7 + github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220808113242-256bcac1fcfe github.com/ElrondNetwork/elrond-go-logger v1.0.7 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index bffa67b7..dc0f52c8 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220729074346-fca5b87ec7c7 h1:31AlKjtaTkdld6Nc28g/hBgV386ft7GbxWwi7Vp0evY= -github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220729074346-fca5b87ec7c7/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220808113242-256bcac1fcfe h1:zKZzyT+D4CrTgJkkvXKROi/xaDN0WhalEt2B5BOZPGc= +github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220808113242-256bcac1fcfe/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7 h1:Ldl1rVS0RGKc1IsW8jIaGCb6Zwei04gsMvyjL05X6mE= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index b58432e8..caed2651 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -14,7 +14,7 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -40,7 +40,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { TimeStamp: 5040, } - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -72,11 +72,11 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { addrForLog := "aaaabbbb" addrForLogHex := hex.EncodeToString([]byte(addrForLog)) - coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + coreAlteredAccounts := map[string]*outport.AlteredAccount{ addrHex: { Address: addrHex, Balance: "0", - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "SSSS-dddd", Balance: "1000", @@ -91,7 +91,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { addrForLogHex: { Address: addrForLogHex, Balance: "0", - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "SSSS-dddd", Balance: "1000", @@ -119,7 +119,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { } esdtDataBytes, _ := json.Marshal(esdtData) - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -146,7 +146,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/collectionsIndex/collections-1.json"), string(genericResponse.Docs[0].Source)) // ################ CREATE SEMI FUNGIBLE TOKEN 2 ########################## - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -179,10 +179,10 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { addr = "aaaabbbbcccccccc" addrHex = hex.EncodeToString([]byte(addr)) - coreAlteredAccounts = map[string]*indexer.AlteredAccount{ + coreAlteredAccounts = map[string]*outport.AlteredAccount{ addrHex: { Address: addrHex, - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "NFT-abcdef", Balance: "0", @@ -198,7 +198,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index ac2588a0..d5561adb 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -14,7 +14,7 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -53,7 +53,7 @@ func TestCreateNFTWithTags(t *testing.T) { esdtDataBytes, _ := json.Marshal(esdtToken) // CREATE A FIRST NFT WITH THE TAGS - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -71,11 +71,11 @@ func TestCreateNFTWithTags(t *testing.T) { }, } - coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + coreAlteredAccounts := map[string]*outport.AlteredAccount{ addrHex: { Address: addrHex, Balance: "0", - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "DESK-abcd", Nonce: 1, @@ -118,7 +118,7 @@ func TestCreateNFTWithTags(t *testing.T) { require.Equal(t, len(ids), tagsChecked) // CREATE A SECOND NFT WITH THE SAME TAGS - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -167,7 +167,7 @@ func TestCreateNFTWithTags(t *testing.T) { esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) require.Nil(t, err) - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", From 784a094e3a82cb5ba02a562862c3c5408d8df0d3 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 12 Aug 2022 16:45:12 +0300 Subject: [PATCH 23/96] remove transactions fee handler --- data/scresult.go | 2 + data/transaction.go | 9 +- factory/indexerFactory.go | 5 - factory/indexerFactory_test.go | 10 -- integrationtests/utils.go | 2 - mock/dbTransactionsHandlerStub.go | 2 +- mock/economicsDataHandlerStub.go | 55 -------- mock/economicsDataMock.go | 121 ------------------ process/elasticProcessor.go | 28 +--- process/elasticProcessor_test.go | 3 - process/factory/elasticProcessorFactory.go | 3 - .../factory/elasticProcessorFactory_test.go | 1 - process/interface.go | 2 +- .../logsevents/informativeLogsProcessor.go | 15 +-- .../informativeLogsProcessor_test.go | 14 +- process/logsevents/logsAndEventsProcessor.go | 6 +- .../logsevents/logsAndEventsProcessor_test.go | 6 - process/logsevents/serialize.go | 6 +- process/transactions/checkers.go | 11 -- process/transactions/checkers_test.go | 10 -- .../transactions/scrsDataToTransactions.go | 57 ++------- .../scrsDataToTransactions_test.go | 9 +- process/transactions/serialize.go | 55 ++++---- process/transactions/serialize_test.go | 22 +--- .../smartContractResultsProcessor.go | 42 +++--- .../smartContractResultsProcessor_test.go | 4 +- process/transactions/transactionDBBuilder.go | 9 +- .../transactions/transactionDBBuilder_test.go | 15 +-- process/transactions/transactionsGrouper.go | 8 +- .../transactions/transactionsGrouper_test.go | 8 +- process/transactions/transactionsProcessor.go | 27 ++-- .../transactionsProcessor_test.go | 11 +- 32 files changed, 114 insertions(+), 464 deletions(-) delete mode 100644 mock/economicsDataHandlerStub.go delete mode 100644 mock/economicsDataMock.go diff --git a/data/scresult.go b/data/scresult.go index 074fa792..5e1871fb 100644 --- a/data/scresult.go +++ b/data/scresult.go @@ -37,4 +37,6 @@ type ScResult struct { CanBeIgnored bool `json:"canBeIgnored,omitempty"` OriginalSender string `json:"originalSender,omitempty"` SenderAddressBytes []byte `json:"-"` + InitialTxGasUsed uint64 `json:"-"` + InitialTxFee string `json:"-"` } diff --git a/data/transaction.go b/data/transaction.go index 3ff5fd2e..531e487b 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -93,7 +93,7 @@ type PreparedResults struct { Receipts []*Receipt AlteredAccts AlteredAccountsHandler TxHashStatus map[string]string - TxHashRefund map[string]*RefundData + TxHashFee map[string]*FeeData } // ResponseTransactions is the structure for the transactions response @@ -108,8 +108,9 @@ type ResponseTransactionDB struct { Source Transaction `json:"_source"` } -// RefundData is the structure that contains data about a refund -type RefundData struct { - Value string +// FeeData is the structure that contains data about transaction fee and gas used +type FeeData struct { + Fee string + GasUsed uint64 Receiver string } diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index 0fbc0653..d43fc67a 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -33,7 +33,6 @@ type ArgsIndexerFactory struct { Hasher hashing.Hasher AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter - TransactionFeeCalculator indexer.FeesProcessorHandler } // NewIndexer will create a new instance of Indexer @@ -88,7 +87,6 @@ func createElasticProcessor(args *ArgsIndexerFactory) (indexer.ElasticProcessor, UseKibana: args.UseKibana, DBClient: databaseClient, Denomination: args.Denomination, - TransactionFeeCalculator: args.TransactionFeeCalculator, IsInImportDBMode: args.IsInImportDBMode, ShardCoordinator: args.ShardCoordinator, EnabledIndexes: args.EnabledIndexes, @@ -117,9 +115,6 @@ func checkDataIndexerParams(arguments *ArgsIndexerFactory) error { if check.IfNil(arguments.Hasher) { return indexer.ErrNilHasher } - if check.IfNil(arguments.TransactionFeeCalculator) { - return indexer.ErrNilTransactionFeeCalculator - } if check.IfNil(arguments.ShardCoordinator) { return indexer.ErrNilShardCoordinator } diff --git a/factory/indexerFactory_test.go b/factory/indexerFactory_test.go index 05941d41..96d72f3f 100644 --- a/factory/indexerFactory_test.go +++ b/factory/indexerFactory_test.go @@ -26,7 +26,6 @@ func createMockIndexerFactoryArgs() *ArgsIndexerFactory { ValidatorPubkeyConverter: &mock.PubkeyConverterMock{}, TemplatesPath: "../testdata", EnabledIndexes: []string{"blocks", "transactions", "miniblocks", "validators", "round", "accounts", "rating"}, - TransactionFeeCalculator: &mock.EconomicsHandlerStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, IsInImportDBMode: false, } @@ -92,15 +91,6 @@ func TestNewIndexerFactory(t *testing.T) { }, exError: indexer.ErrNilUrl, }, - { - name: "NilTransactionFeeCalculator", - argsFunc: func() *ArgsIndexerFactory { - args := createMockIndexerFactoryArgs() - args.TransactionFeeCalculator = nil - return args - }, - exError: indexer.ErrNilTransactionFeeCalculator, - }, { name: "NilShardCoordinator", argsFunc: func() *ArgsIndexerFactory { diff --git a/integrationtests/utils.go b/integrationtests/utils.go index c03d7499..28ec2e4b 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -35,7 +35,6 @@ func createESClient(url string) (process.DatabaseClientHandler, error) { func CreateElasticProcessor( esClient process.DatabaseClientHandler, shardCoordinator indexer.ShardCoordinator, - feeProcessor indexer.FeesProcessorHandler, ) (indexer.ElasticProcessor, error) { args := factory.ArgElasticProcessorFactory{ Marshalizer: &mock.MarshalizerMock{}, @@ -44,7 +43,6 @@ func CreateElasticProcessor( ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32), DBClient: esClient, ShardCoordinator: shardCoordinator, - TransactionFeeCalculator: feeProcessor, EnabledIndexes: []string{indexer.TransactionsIndex, indexer.LogsIndex, indexer.AccountsESDTIndex, indexer.ScResultsIndex, indexer.ReceiptsIndex, indexer.BlockIndex, indexer.AccountsIndex, indexer.TokensIndex, indexer.TagsIndex, indexer.CollectionsIndex}, Denomination: 18, diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index b9151c96..f25f3d9a 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -14,7 +14,7 @@ type DBTransactionProcessorStub struct { SerializeScResultsCalled func(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } -func (tps *DBTransactionProcessorStub) SerializeTransactionWithRefund(_ map[string]*data.Transaction, _ map[string]*data.RefundData, _ *data.BufferSlice, _ string) error { +func (tps *DBTransactionProcessorStub) SerializeTransactionsFeeData(_ map[string]*data.FeeData, _ *data.BufferSlice, _ string) error { return nil } diff --git a/mock/economicsDataHandlerStub.go b/mock/economicsDataHandlerStub.go deleted file mode 100644 index a6f1c5d8..00000000 --- a/mock/economicsDataHandlerStub.go +++ /dev/null @@ -1,55 +0,0 @@ -package mock - -import ( - "math/big" - - coreData "github.com/ElrondNetwork/elrond-go-core/data" -) - -// EconomicsHandlerStub - -type EconomicsHandlerStub struct { - ComputeGasLimitCalled func(tx coreData.TransactionWithFeeHandler) uint64 - MinGasLimitCalled func() uint64 - ComputeGasUsedAndFeeBasedOnRefundValueCalled func(tx coreData.TransactionWithFeeHandler, refundValue *big.Int) (uint64, *big.Int) - ComputeTxFeeBasedOnGasUsedCalled func(tx coreData.TransactionWithFeeHandler, gasUsed uint64) *big.Int - ComputeMoveBalanceGasUsedCalled func(tx coreData.TransactionWithFeeHandler) uint64 -} - -// MinGasLimit - -func (e *EconomicsHandlerStub) MinGasLimit() uint64 { - if e.MinGasLimitCalled != nil { - return e.MinGasLimitCalled() - } - return 0 -} - -// ComputeGasLimit - -func (e *EconomicsHandlerStub) ComputeGasLimit(tx coreData.TransactionWithFeeHandler) uint64 { - if e.ComputeGasLimitCalled != nil { - return e.ComputeGasLimitCalled(tx) - } - return 0 -} - -// ComputeGasUsedAndFeeBasedOnRefundValue - -func (e *EconomicsHandlerStub) ComputeGasUsedAndFeeBasedOnRefundValue(tx coreData.TransactionWithFeeHandler, refundValue *big.Int) (uint64, *big.Int) { - if e.ComputeGasUsedAndFeeBasedOnRefundValueCalled != nil { - return e.ComputeGasUsedAndFeeBasedOnRefundValueCalled(tx, refundValue) - } - - return 0, nil -} - -// ComputeTxFeeBasedOnGasUsed - -func (e *EconomicsHandlerStub) ComputeTxFeeBasedOnGasUsed(tx coreData.TransactionWithFeeHandler, gasUsed uint64) *big.Int { - if e.ComputeTxFeeBasedOnGasUsedCalled != nil { - return e.ComputeTxFeeBasedOnGasUsedCalled(tx, gasUsed) - } - - return nil -} - -// IsInterfaceNil returns true if there is no value under the interface -func (e *EconomicsHandlerStub) IsInterfaceNil() bool { - return e == nil -} diff --git a/mock/economicsDataMock.go b/mock/economicsDataMock.go deleted file mode 100644 index 6f24752a..00000000 --- a/mock/economicsDataMock.go +++ /dev/null @@ -1,121 +0,0 @@ -package mock - -import ( - "math/big" - - "github.com/ElrondNetwork/elrond-go-core/core" - coreData "github.com/ElrondNetwork/elrond-go-core/data" - logger "github.com/ElrondNetwork/elrond-go-logger" -) - -var log = logger.GetOrCreate("mock/economicsData") - -const ( - minGasLimit = uint64(50000) - gasPerDataByte = uint64(1500) - gasPriceModifier = float64(0.01) -) - -// EconomicsHandlerMock - -type EconomicsHandlerMock struct { -} - -// MinGasLimit - -func (e *EconomicsHandlerMock) MinGasLimit() uint64 { - return minGasLimit -} - -// ComputeGasLimit - -func (e *EconomicsHandlerMock) ComputeGasLimit(tx coreData.TransactionWithFeeHandler) uint64 { - gasLimit := minGasLimit - - dataLen := uint64(len(tx.GetData())) - gasLimit += dataLen * gasPerDataByte - - return gasLimit -} - -// ComputeGasUsedAndFeeBasedOnRefundValue - -func (e *EconomicsHandlerMock) ComputeGasUsedAndFeeBasedOnRefundValue(tx coreData.TransactionWithFeeHandler, refundValue *big.Int) (uint64, *big.Int) { - if refundValue.Cmp(big.NewInt(0)) == 0 { - txFee := e.ComputeTxFee(tx) - return tx.GetGasLimit(), txFee - } - - txFee := e.ComputeTxFee(tx) - - txFee = big.NewInt(0).Sub(txFee, refundValue) - - moveBalanceGasUnits := e.ComputeGasLimit(tx) - moveBalanceFee := e.computeMoveBalanceFee(tx) - - scOpFee := big.NewInt(0).Sub(txFee, moveBalanceFee) - gasPriceForProcessing := big.NewInt(0).SetUint64(e.GasPriceForProcessing(tx)) - scOpGasUnits := big.NewInt(0).Div(scOpFee, gasPriceForProcessing) - - gasUsed := moveBalanceGasUnits + scOpGasUnits.Uint64() - - return gasUsed, txFee -} - -// ComputeTxFeeBasedOnGasUsed - -func (e *EconomicsHandlerMock) ComputeTxFeeBasedOnGasUsed(tx coreData.TransactionWithFeeHandler, gasUsed uint64) *big.Int { - moveBalanceGasLimit := e.ComputeGasLimit(tx) - moveBalanceFee := e.computeMoveBalanceFee(tx) - if gasUsed <= moveBalanceGasLimit { - return moveBalanceFee - } - - computeFeeForProcessing := e.ComputeFeeForProcessing(tx, gasUsed-moveBalanceGasLimit) - txFee := big.NewInt(0).Add(moveBalanceFee, computeFeeForProcessing) - - return txFee -} - -func (e *EconomicsHandlerMock) computeMoveBalanceFee(tx coreData.TransactionWithFeeHandler) *big.Int { - return core.SafeMul(tx.GetGasPrice(), e.ComputeGasLimit(tx)) -} - -// IsInterfaceNil returns true if there is no value under the interface -func (e *EconomicsHandlerMock) IsInterfaceNil() bool { - return e == nil -} - -// ComputeFeeForProcessing will compute the fee using the gas price modifier, the gas to use and the actual gas price -func (e *EconomicsHandlerMock) ComputeFeeForProcessing(tx coreData.TransactionWithFeeHandler, gasToUse uint64) *big.Int { - gasPrice := e.GasPriceForProcessing(tx) - return core.SafeMul(gasPrice, gasToUse) -} - -// GasPriceForProcessing computes the price for the gas in addition to balance movement and data -func (e *EconomicsHandlerMock) GasPriceForProcessing(tx coreData.TransactionWithFeeHandler) uint64 { - return uint64(float64(tx.GetGasPrice()) * gasPriceModifier) -} - -func (e *EconomicsHandlerMock) ComputeTxFee(tx coreData.TransactionWithFeeHandler) *big.Int { - gasLimitForMoveBalance, difference := e.SplitTxGasInCategories(tx) - moveBalanceFee := core.SafeMul(tx.GetGasPrice(), gasLimitForMoveBalance) - if tx.GetGasLimit() <= gasLimitForMoveBalance { - return moveBalanceFee - } - - extraFee := e.ComputeFeeForProcessing(tx, difference) - moveBalanceFee.Add(moveBalanceFee, extraFee) - return moveBalanceFee -} - -// SplitTxGasInCategories returns the gas split per categories -func (e *EconomicsHandlerMock) SplitTxGasInCategories(tx coreData.TransactionWithFeeHandler) (gasLimitMove, gasLimitProcess uint64) { - var err error - gasLimitMove = e.ComputeGasLimit(tx) - gasLimitProcess, err = core.SafeSubUint64(tx.GetGasLimit(), gasLimitMove) - if err != nil { - log.Error("SplitTxGasInCategories - insufficient gas for move", - "providedGas", tx.GetGasLimit(), - "computedMinimumRequired", gasLimitMove, - "dataLen", len(tx.GetData()), - ) - } - - return -} diff --git a/process/elasticProcessor.go b/process/elasticProcessor.go index eb3f7fdf..d2cac166 100644 --- a/process/elasticProcessor.go +++ b/process/elasticProcessor.go @@ -365,7 +365,7 @@ func (ei *elasticProcessor) SaveTransactions( return err } - err = ei.indexTransactionsWithRefund(preparedResults.TxHashRefund, buffers) + err = ei.indexTransactionsFeeData(preparedResults.TxHashFee, buffers) if err != nil { return err } @@ -450,32 +450,12 @@ func (ei *elasticProcessor) prepareAndIndexDelegators(delegators map[string]*dat return ei.logsAndEventsProc.SerializeDelegators(delegators, buffSlice, elasticIndexer.DelegatorsIndex) } -func (ei *elasticProcessor) indexTransactionsWithRefund(txsHashRefund map[string]*data.RefundData, buffSlice *data.BufferSlice) error { - if len(txsHashRefund) == 0 { +func (ei *elasticProcessor) indexTransactionsFeeData(txsHashFeeData map[string]*data.FeeData, buffSlice *data.BufferSlice) error { + if len(txsHashFeeData) == 0 { return nil } - txsHashes := make([]string, len(txsHashRefund)) - for txHash := range txsHashRefund { - txsHashes = append(txsHashes, txHash) - } - - responseTransactions := &data.ResponseTransactions{} - err := ei.elasticClient.DoMultiGet(txsHashes, elasticIndexer.TransactionsIndex, true, responseTransactions) - if err != nil { - return err - } - - txsFromDB := make(map[string]*data.Transaction) - for idx := 0; idx < len(responseTransactions.Docs); idx++ { - txRes := responseTransactions.Docs[idx] - if !txRes.Found { - continue - } - - txsFromDB[txRes.ID] = &txRes.Source - } - return ei.transactionsProc.SerializeTransactionWithRefund(txsFromDB, txsHashRefund, buffSlice, elasticIndexer.TransactionsIndex) + return ei.transactionsProc.SerializeTransactionsFeeData(txsHashFeeData, buffSlice, elasticIndexer.TransactionsIndex) } func (ei *elasticProcessor) prepareAndIndexLogs(logsAndEvents []*coreData.LogData, timestamp uint64, buffSlice *data.BufferSlice) error { diff --git a/process/elasticProcessor_test.go b/process/elasticProcessor_test.go index 4ddd3008..2a2e9dcb 100644 --- a/process/elasticProcessor_test.go +++ b/process/elasticProcessor_test.go @@ -60,7 +60,6 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: balanceConverter, Hasher: &mock.HasherMock{}, - TxFeeCalculator: &mock.EconomicsHandlerStub{}, } lp, _ := logsevents.NewLogsAndEventsProcessor(args) op, _ := operations.NewOperationsProcessor(false, &mock.ShardCoordinatorMock{}) @@ -420,7 +419,6 @@ func TestElasticseachSaveTransactions(t *testing.T) { args := &transactions.ArgsTransactionProcessor{ AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), - TxFeeCalculator: &mock.EconomicsHandlerStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, @@ -572,7 +570,6 @@ func TestElasticProcessor_RemoveTransactions(t *testing.T) { args := &transactions.ArgsTransactionProcessor{ AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), - TxFeeCalculator: &mock.EconomicsHandlerStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, diff --git a/process/factory/elasticProcessorFactory.go b/process/factory/elasticProcessorFactory.go index 7450ca5e..87a69e77 100644 --- a/process/factory/elasticProcessorFactory.go +++ b/process/factory/elasticProcessorFactory.go @@ -26,7 +26,6 @@ type ArgElasticProcessorFactory struct { ValidatorPubkeyConverter core.PubkeyConverter DBClient processIndexer.DatabaseClientHandler ShardCoordinator indexer.ShardCoordinator - TransactionFeeCalculator indexer.FeesProcessorHandler EnabledIndexes []string Denomination int BulkRequestMaxSize int @@ -81,7 +80,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.Elast argsTxsProc := &transactions.ArgsTransactionProcessor{ AddressPubkeyConverter: arguments.AddressPubkeyConverter, - TxFeeCalculator: arguments.TransactionFeeCalculator, ShardCoordinator: arguments.ShardCoordinator, Hasher: arguments.Hasher, Marshalizer: arguments.Marshalizer, @@ -98,7 +96,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.Elast Marshalizer: arguments.Marshalizer, BalanceConverter: balanceConverter, Hasher: arguments.Hasher, - TxFeeCalculator: arguments.TransactionFeeCalculator, } logsAndEventsProc, err := logsevents.NewLogsAndEventsProcessor(argsLogsAndEventsProc) if err != nil { diff --git a/process/factory/elasticProcessorFactory_test.go b/process/factory/elasticProcessorFactory_test.go index d3dcfff3..5ac03546 100644 --- a/process/factory/elasticProcessorFactory_test.go +++ b/process/factory/elasticProcessorFactory_test.go @@ -16,7 +16,6 @@ func TestCreateElasticProcessor(t *testing.T) { ValidatorPubkeyConverter: &mock.PubkeyConverterMock{}, DBClient: &mock.DatabaseWriterStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, - TransactionFeeCalculator: &mock.EconomicsHandlerStub{}, EnabledIndexes: []string{"blocks"}, Denomination: 1, IsInImportDBMode: false, diff --git a/process/interface.go b/process/interface.go index d67edb00..f5716d77 100644 --- a/process/interface.go +++ b/process/interface.go @@ -70,7 +70,7 @@ type DBTransactionsHandler interface { SerializeReceipts(receipts []*data.Receipt, buffSlice *data.BufferSlice, index string) error SerializeTransactions(transactions []*data.Transaction, txHashStatus map[string]string, selfShardID uint32, buffSlice *data.BufferSlice, index string) error - SerializeTransactionWithRefund(txs map[string]*data.Transaction, txHashRefund map[string]*data.RefundData, buffSlice *data.BufferSlice, index string) error + SerializeTransactionsFeeData(txHashRefund map[string]*data.FeeData, buffSlice *data.BufferSlice, index string) error SerializeScResults(scResults []*data.ScResult, buffSlice *data.BufferSlice, index string) error } diff --git a/process/logsevents/informativeLogsProcessor.go b/process/logsevents/informativeLogsProcessor.go index 9d559b6b..4d6b496c 100644 --- a/process/logsevents/informativeLogsProcessor.go +++ b/process/logsevents/informativeLogsProcessor.go @@ -1,9 +1,6 @@ package logsevents import ( - "math/big" - - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elrond-go-core/data/transaction" ) @@ -13,17 +10,15 @@ const ( ) type informativeLogsProcessor struct { - operations map[string]struct{} - txFeeCalculator indexer.FeesProcessorHandler + operations map[string]struct{} } -func newInformativeLogsProcessor(txFeeCalculator indexer.FeesProcessorHandler) *informativeLogsProcessor { +func newInformativeLogsProcessor() *informativeLogsProcessor { return &informativeLogsProcessor{ operations: map[string]struct{}{ writeLogOperation: {}, signalErrorOperation: {}, }, - txFeeCalculator: txFeeCalculator, } } @@ -44,16 +39,10 @@ func (ilp *informativeLogsProcessor) processEvent(args *argsProcessEvent) argOut switch identifier { case writeLogOperation: { - gasLimit, fee := ilp.txFeeCalculator.ComputeGasUsedAndFeeBasedOnRefundValue(tx, big.NewInt(0)) - tx.GasUsed = gasLimit - tx.Fee = fee.String() tx.Status = transaction.TxStatusSuccess.String() } case signalErrorOperation: { - tx.GasUsed = tx.GasLimit - fee := ilp.txFeeCalculator.ComputeTxFeeBasedOnGasUsed(tx, tx.GasLimit) - tx.Fee = fee.String() tx.Status = transaction.TxStatusFail.String() } } diff --git a/process/logsevents/informativeLogsProcessor_test.go b/process/logsevents/informativeLogsProcessor_test.go index c4398ad0..3caa7008 100644 --- a/process/logsevents/informativeLogsProcessor_test.go +++ b/process/logsevents/informativeLogsProcessor_test.go @@ -4,14 +4,12 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) func TestInformativeShouldIgnoreLog(t *testing.T) { - feeHandler := &mock.EconomicsHandlerMock{} - informativeLogsProc := newInformativeLogsProcessor(feeHandler) + informativeLogsProc := newInformativeLogsProcessor() event := &transaction.Event{ Address: []byte("addr"), @@ -52,14 +50,11 @@ func TestInformativeLogsProcessorWriteLog(t *testing.T) { txHashHexEncoded: hexEncodedTxHash, } - feeHandler := &mock.EconomicsHandlerMock{} - informativeLogsProc := newInformativeLogsProcessor(feeHandler) + informativeLogsProc := newInformativeLogsProcessor() res := informativeLogsProc.processEvent(args) require.Equal(t, transaction.TxStatusSuccess.String(), tx.Status) - require.Equal(t, tx.GasLimit, tx.GasUsed) - require.Equal(t, "7083500000", tx.Fee) require.True(t, res.processed) } @@ -88,13 +83,10 @@ func TestInformativeLogsProcessorSignalError(t *testing.T) { txHashHexEncoded: hexEncodedTxHash, } - feeHandler := &mock.EconomicsHandlerMock{} - informativeLogsProc := newInformativeLogsProcessor(feeHandler) + informativeLogsProc := newInformativeLogsProcessor() res := informativeLogsProc.processEvent(args) require.Equal(t, transaction.TxStatusFail.String(), tx.Status) - require.Equal(t, tx.GasLimit, tx.GasUsed) - require.Equal(t, "6041000000", tx.Fee) require.Equal(t, true, res.processed) } diff --git a/process/logsevents/logsAndEventsProcessor.go b/process/logsevents/logsAndEventsProcessor.go index 929063c7..7fc352ec 100644 --- a/process/logsevents/logsAndEventsProcessor.go +++ b/process/logsevents/logsAndEventsProcessor.go @@ -20,7 +20,6 @@ type ArgsLogsAndEventsProcessor struct { Marshalizer marshal.Marshalizer BalanceConverter elasticIndexer.BalanceConverter Hasher hashing.Hasher - TxFeeCalculator elasticIndexer.FeesProcessorHandler } type logsAndEventsProcessor struct { @@ -63,9 +62,6 @@ func checkArgsLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) error { if check.IfNil(args.Hasher) { return elasticIndexer.ErrNilHasher } - if check.IfNil(args.TxFeeCalculator) { - return elasticIndexer.ErrNilTransactionFeeCalculator - } return nil } @@ -74,7 +70,7 @@ func createEventsProcessors(args *ArgsLogsAndEventsProcessor) []eventsProcessor nftsProc := newNFTsProcessor(args.ShardCoordinator, args.PubKeyConverter, args.Marshalizer) fungibleProc := newFungibleESDTProcessor(args.PubKeyConverter, args.ShardCoordinator) scDeploysProc := newSCDeploysProcessor(args.PubKeyConverter) - informativeProc := newInformativeLogsProcessor(args.TxFeeCalculator) + informativeProc := newInformativeLogsProcessor() updateNFTProc := newNFTsPropertiesProcessor(args.PubKeyConverter) esdtPropProc := newEsdtPropertiesProcessor(args.PubKeyConverter) diff --git a/process/logsevents/logsAndEventsProcessor_test.go b/process/logsevents/logsAndEventsProcessor_test.go index 1e336cb3..9e8691d8 100644 --- a/process/logsevents/logsAndEventsProcessor_test.go +++ b/process/logsevents/logsAndEventsProcessor_test.go @@ -23,7 +23,6 @@ func createMockArgs() *ArgsLogsAndEventsProcessor { Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: balanceConverter, Hasher: &mock.HasherMock{}, - TxFeeCalculator: &mock.EconomicsHandlerStub{}, } } @@ -55,11 +54,6 @@ func TestNewLogsAndEventsProcessor(t *testing.T) { _, err = NewLogsAndEventsProcessor(args) require.Equal(t, elasticIndexer.ErrNilHasher, err) - args = createMockArgs() - args.TxFeeCalculator = nil - _, err = NewLogsAndEventsProcessor(args) - require.Equal(t, elasticIndexer.ErrNilTransactionFeeCalculator, err) - args = createMockArgs() proc, err := NewLogsAndEventsProcessor(args) require.NotNil(t, proc) diff --git a/process/logsevents/serialize.go b/process/logsevents/serialize.go index f68dbf6a..aeb66c3d 100644 --- a/process/logsevents/serialize.go +++ b/process/logsevents/serialize.go @@ -12,7 +12,7 @@ import ( ) // SerializeLogs will serialize the provided logs in a way that Elastic Search expects a bulk request -func (logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data.BufferSlice, index string) error { +func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data.BufferSlice, index string) error { for _, lg := range logs { meta := []byte(fmt.Sprintf(`{ "index" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(lg.ID), "\n")) serializedData, errMarshal := json.Marshal(lg) @@ -30,7 +30,7 @@ func (logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data.B } // SerializeSCDeploys will serialize the provided smart contract deploys in a way that Elastic Search expects a bulk request -func (logsAndEventsProcessor) SerializeSCDeploys(deploys map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice, index string) error { +func (*logsAndEventsProcessor) SerializeSCDeploys(deploys map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice, index string) error { for scAddr, deployInfo := range deploys { meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(scAddr), "\n")) @@ -83,7 +83,7 @@ func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) { } // SerializeTokens will serialize the provided tokens' data in a way that Elasticsearch expects a bulk request -func (logsAndEventsProcessor) SerializeTokens(tokens []*data.TokenInfo, updateNFTData []*data.NFTDataUpdate, buffSlice *data.BufferSlice, index string) error { +func (*logsAndEventsProcessor) SerializeTokens(tokens []*data.TokenInfo, updateNFTData []*data.NFTDataUpdate, buffSlice *data.BufferSlice, index string) error { for _, tokenData := range tokens { meta, serializedData, err := serializeToken(tokenData, index) if err != nil { diff --git a/process/transactions/checkers.go b/process/transactions/checkers.go index c3329752..0f9587c2 100644 --- a/process/transactions/checkers.go +++ b/process/transactions/checkers.go @@ -36,9 +36,6 @@ func checkTxsProcessorArg(args *ArgsTransactionProcessor) error { if check.IfNil(args.AddressPubkeyConverter) { return elasticIndexer.ErrNilPubkeyConverter } - if check.IfNil(args.TxFeeCalculator) { - return elasticIndexer.ErrNilTransactionFeeCalculator - } return nil } @@ -77,14 +74,6 @@ func isSCRForSenderWithRefund(dbScResult *data.ScResult, tx *data.Transaction) b return isFromCurrentTx && isForSender && isRightNonce && isScrDataOk } -func isRefundForRelayed(dbScResult *data.ScResult, tx *data.Transaction) bool { - isForRelayed := dbScResult.ReturnMessage == data.GasRefundForRelayerMessage - isForSender := dbScResult.Receiver == tx.Sender - differentHash := dbScResult.OriginalTxHash != dbScResult.PrevTxHash - - return isForRelayed && isForSender && differentHash -} - func isDataOk(data []byte) bool { dataFieldStr := "@" + okHexEncoded diff --git a/process/transactions/checkers_test.go b/process/transactions/checkers_test.go index 5c477c93..051e12c5 100644 --- a/process/transactions/checkers_test.go +++ b/process/transactions/checkers_test.go @@ -19,7 +19,6 @@ import ( func createMockArgs() *ArgsTransactionProcessor { return &ArgsTransactionProcessor{ AddressPubkeyConverter: &mock.PubkeyConverterMock{}, - TxFeeCalculator: &mock.EconomicsHandlerStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, @@ -44,15 +43,6 @@ func TestNewTransactionsProcessor(t *testing.T) { }, exErr: elasticIndexer.ErrNilPubkeyConverter, }, - { - name: "NilTxFeeCalculator", - args: func() *ArgsTransactionProcessor { - args := createMockArgs() - args.TxFeeCalculator = nil - return args - }, - exErr: elasticIndexer.ErrNilTransactionFeeCalculator, - }, { name: "NilShardCoordinator", args: func() *ArgsTransactionProcessor { diff --git a/process/transactions/scrsDataToTransactions.go b/process/transactions/scrsDataToTransactions.go index 4910bc9c..4971b724 100644 --- a/process/transactions/scrsDataToTransactions.go +++ b/process/transactions/scrsDataToTransactions.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "strings" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/transaction" @@ -16,13 +15,11 @@ const ( ) type scrsDataToTransactions struct { - retCodes []string - txFeeCalculator indexer.FeesProcessorHandler + retCodes []string } -func newScrsDataToTransactions(txFeeCalculator indexer.FeesProcessorHandler) *scrsDataToTransactions { +func newScrsDataToTransactions() *scrsDataToTransactions { return &scrsDataToTransactions{ - txFeeCalculator: txFeeCalculator, retCodes: []string{ vmcommon.FunctionNotFound.String(), vmcommon.FunctionWrongSignature.String(), @@ -53,36 +50,12 @@ func (st *scrsDataToTransactions) attachSCRsToTransactionsAndReturnSCRsWithoutTx continue } - st.addScResultInfoIntoTx(scr, tx) + tx.SmartContractResults = append(tx.SmartContractResults, scr) } return scrsWithoutTx } -func (st *scrsDataToTransactions) addScResultInfoIntoTx(dbScResult *data.ScResult, tx *data.Transaction) { - tx.SmartContractResults = append(tx.SmartContractResults, dbScResult) - isRelayedTxFirstSCR := isRelayedTx(tx) && len(tx.SmartContractResults) == 1 - if isRelayedTxFirstSCR { - tx.GasUsed = tx.GasLimit - fee := st.txFeeCalculator.ComputeTxFeeBasedOnGasUsed(tx, tx.GasUsed) - tx.Fee = fee.String() - } - - // ignore invalid transaction because status and gas fields was already set - if tx.Status == transaction.TxStatusInvalid.String() { - return - } - - if isSCRForSenderWithRefund(dbScResult, tx) || isRefundForRelayed(dbScResult, tx) { - refundValue := stringValueToBigInt(dbScResult.Value) - gasUsed, fee := st.txFeeCalculator.ComputeGasUsedAndFeeBasedOnRefundValue(tx, refundValue) - tx.GasUsed = gasUsed - tx.Fee = fee.String() - } - - return -} - func (st *scrsDataToTransactions) processTransactionsAfterSCRsWereAttached(transactions map[string]*data.Transaction) { for _, tx := range transactions { if len(tx.SmartContractResults) == 0 { @@ -109,10 +82,6 @@ func (st *scrsDataToTransactions) fillTxWithSCRsFields(tx *data.Transaction) { return } - tx.GasUsed = tx.GasLimit - fee := st.txFeeCalculator.ComputeTxFeeBasedOnGasUsed(tx, tx.GasUsed) - tx.Fee = fee.String() - if hasCrossShardPendingTransfer(tx) { return } @@ -165,13 +134,14 @@ func hasCrossShardPendingTransfer(tx *data.Transaction) bool { return false } -func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) (map[string]string, map[string]*data.RefundData) { +func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) (map[string]string, map[string]*data.FeeData) { txHashStatus := make(map[string]string) - txHashRefund := make(map[string]*data.RefundData) + txHashRefund := make(map[string]*data.FeeData) for _, scr := range scrs { - if isSCRWithRefund(scr) { - txHashRefund[scr.OriginalTxHash] = &data.RefundData{ - Value: scr.Value, + if scr.InitialTxGasUsed != 0 { + txHashRefund[scr.OriginalTxHash] = &data.FeeData{ + Fee: scr.InitialTxFee, + GasUsed: scr.InitialTxGasUsed, Receiver: scr.Receiver, } } @@ -186,15 +156,6 @@ func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) (m return txHashStatus, txHashRefund } -func isSCRWithRefund(scr *data.ScResult) bool { - hasRefund := scr.Value != "0" && scr.Value != emptyString - isSuccessful := isScResultSuccessful(scr.Data) - isRefundForRelayTxSender := scr.ReturnMessage == data.GasRefundForRelayerMessage - ok := isSuccessful || isRefundForRelayTxSender - - return ok && scr.OriginalTxHash != scr.PrevTxHash && hasRefund -} - func isESDTNFTTransferWithUserError(scrData string) bool { splitData := strings.Split(scrData, data.AtSeparator) isMultiTransferOrNFTTransfer := splitData[0] == core.BuiltInFunctionESDTNFTTransfer || splitData[0] == core.BuiltInFunctionMultiESDTNFTTransfer diff --git a/process/transactions/scrsDataToTransactions_test.go b/process/transactions/scrsDataToTransactions_test.go index 3d247ccc..5c1770d4 100644 --- a/process/transactions/scrsDataToTransactions_test.go +++ b/process/transactions/scrsDataToTransactions_test.go @@ -5,14 +5,13 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/stretchr/testify/require" ) func TestAttachSCRsToTransactionsAndReturnSCRsWithoutTx(t *testing.T) { t.Parallel() - scrsDataToTxs := newScrsDataToTransactions(&mock.EconomicsHandlerMock{}) + scrsDataToTxs := newScrsDataToTransactions() txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -24,6 +23,8 @@ func TestAttachSCRsToTransactionsAndReturnSCRsWithoutTx(t *testing.T) { GasLimit: 10000000, GasPrice: 1000000000, Data: []byte("callSomething"), + GasUsed: 5963500, + Fee: "128440000000000", } tx2 := &data.Transaction{} txs := map[string]*data.Transaction{ @@ -57,7 +58,7 @@ func TestAttachSCRsToTransactionsAndReturnSCRsWithoutTx(t *testing.T) { func TestProcessTransactionsAfterSCRsWereAttached(t *testing.T) { t.Parallel() - scrsDataToTxs := newScrsDataToTransactions(&mock.EconomicsHandlerMock{}) + scrsDataToTxs := newScrsDataToTransactions() txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -74,6 +75,8 @@ func TestProcessTransactionsAfterSCRsWereAttached(t *testing.T) { ReturnMessage: "user error", }, }, + GasUsed: 10000000, + Fee: "168805000000000", } tx2 := &data.Transaction{} txs := map[string]*data.Transaction{ diff --git a/process/transactions/serialize.go b/process/transactions/serialize.go index e77571a5..49d48ce4 100644 --- a/process/transactions/serialize.go +++ b/process/transactions/serialize.go @@ -3,7 +3,6 @@ package transactions import ( "encoding/json" "fmt" - "math/big" "strings" "github.com/ElrondNetwork/elastic-indexer-go/converters" @@ -11,7 +10,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" ) -// SerializeScResults will serialize the provided smart contract results in a way that Elastic Search expects a bulk request +// SerializeScResults will serialize the provided smart contract results in a way that ElasticSearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeScResults(scResults []*data.ScResult, buffSlice *data.BufferSlice, index string) error { for _, sc := range scResults { meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(sc.Hash), "\n")) @@ -29,7 +28,7 @@ func (tdp *txsDatabaseProcessor) SerializeScResults(scResults []*data.ScResult, return nil } -// SerializeReceipts will serialize the receipts in a way that Elastic Search expects a bulk request +// SerializeReceipts will serialize the receipts in a way that ElasticSearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeReceipts(receipts []*data.Receipt, buffSlice *data.BufferSlice, index string) error { for _, rec := range receipts { meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(rec.Hash), "\n")) @@ -47,38 +46,28 @@ func (tdp *txsDatabaseProcessor) SerializeReceipts(receipts []*data.Receipt, buf return nil } -// SerializeTransactionWithRefund will serialize transaction based on refund -func (tdp *txsDatabaseProcessor) SerializeTransactionWithRefund( - txs map[string]*data.Transaction, - txHashRefund map[string]*data.RefundData, - buffSlice *data.BufferSlice, - index string, -) error { - for txHash, tx := range txs { - refundForTx, ok := txHashRefund[txHash] - if !ok { - continue - } - - if refundForTx.Receiver != tx.Sender { - continue - } +// SerializeTransactionsFeeData will serialize transactions fee data +func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[string]*data.FeeData, buffSlice *data.BufferSlice, index string) error { + for txHash, feeData := range txHashRefund { + meta := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, converters.JsonEscape(txHash), "\n")) + codeToExecute := ` + if ('create' == ctx.op) { + ctx.op = 'noop' + } else { + ctx._source.fee = params.fee; + ctx._source.gasUsed = params.gasUsed; + } +` - refundValueBig, ok := big.NewInt(0).SetString(refundForTx.Value, 10) - if !ok { - continue - } - gasUsed, fee := tdp.txFeeCalculator.ComputeGasUsedAndFeeBasedOnRefundValue(tx, refundValueBig) - tx.GasUsed = gasUsed - tx.Fee = fee.String() - - meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(txHash), "\n")) - serializedData, errPrepare := json.Marshal(tx) - if errPrepare != nil { - return errPrepare - } + serializedDataStr := fmt.Sprintf(`{"scripted_upsert": true, "script": {`+ + `"source": "%s",`+ + `"lang": "painless",`+ + `"params": {"fee": "%s", "gasUsed": %d}},`+ + `"upsert": {}}`, + converters.FormatPainlessSource(codeToExecute), feeData.Fee, feeData.GasUsed, + ) - err := buffSlice.PutData(meta, serializedData) + err := buffSlice.PutData(meta, []byte(serializedDataStr)) if err != nil { return err } diff --git a/process/transactions/serialize_test.go b/process/transactions/serialize_test.go index 82114c3b..0cc96e16 100644 --- a/process/transactions/serialize_test.go +++ b/process/transactions/serialize_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/stretchr/testify/require" ) @@ -128,29 +127,20 @@ func TestSerializeTransactionsCrossShardTxDestination(t *testing.T) { func TestTxsDatabaseProcessor_SerializeTransactionWithRefund(t *testing.T) { t.Parallel() - txs := map[string]*data.Transaction{ + txHashRefund := map[string]*data.FeeData{ "txHash": { - Sender: "sender", - Receiver: "receiver", - GasLimit: 150000000, - GasPrice: 1000000000, - }, - } - txHashRefund := map[string]*data.RefundData{ - "txHash": { - Value: "101676480000000", + Fee: "100000", + GasUsed: 5000, Receiver: "sender", }, } buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - err := (&txsDatabaseProcessor{ - txFeeCalculator: &mock.EconomicsHandlerMock{}, - }).SerializeTransactionWithRefund(txs, txHashRefund, buffSlice, "transactions") + err := (&txsDatabaseProcessor{}).SerializeTransactionsFeeData(txHashRefund, buffSlice, "transactions") require.Nil(t, err) - expectedBuff := `{ "index" : { "_index": "transactions", "_id" : "txHash" } } -{"miniBlockHash":"","nonce":0,"round":0,"value":"","receiver":"receiver","sender":"sender","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":150000000,"gasUsed":139832352,"fee":"1447823520000000","data":null,"signature":"","timestamp":0,"status":"","searchOrder":0} + expectedBuff := `{"update":{ "_index":"transactions","_id":"txHash"}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx.op = 'noop'} else {ctx._source.fee = params.fee;ctx._source.gasUsed = params.gasUsed;}","lang": "painless","params": {"fee": "100000", "gasUsed": 5000}},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } diff --git a/process/transactions/smartContractResultsProcessor.go b/process/transactions/smartContractResultsProcessor.go index 13633b42..420ee905 100644 --- a/process/transactions/smartContractResultsProcessor.go +++ b/process/transactions/smartContractResultsProcessor.go @@ -2,6 +2,7 @@ package transactions import ( "encoding/hex" + "math/big" "strconv" "time" @@ -47,48 +48,35 @@ func (proc *smartContractResultsProcessor) processSCRs( txsHandler map[string]data.TransactionHandlerWithGasUsedAndFee, ) []*indexerData.ScResult { allSCRs := make([]*indexerData.ScResult, 0, len(txsHandler)) - scrs := convertHandlerMap(txsHandler) for _, mb := range body.MiniBlocks { if mb.Type != block.SmartContractResultBlock { continue } - indexerSCRs := proc.processSCRsFromMiniblock(header, mb, scrs) + indexerSCRs := proc.processSCRsFromMiniblock(header, mb, txsHandler) allSCRs = append(allSCRs, indexerSCRs...) } selfShardID := proc.shardCoordinator.SelfId() - for scrHash, noMBScr := range scrs { - indexerScr := proc.prepareSmartContractResult([]byte(scrHash), nil, noMBScr, header, selfShardID, selfShardID) - - allSCRs = append(allSCRs, indexerScr) - } - - return allSCRs -} - -func convertHandlerMap(txsHandler map[string]data.TransactionHandlerWithGasUsedAndFee) map[string]*smartContractResult.SmartContractResult { - scrs := make(map[string]*smartContractResult.SmartContractResult, len(txsHandler)) - for txHandlerHash, txHandler := range txsHandler { - scr, ok := txHandler.GetTxHandler().(*smartContractResult.SmartContractResult) + for scrHash, noMBScr := range txsHandler { + scr, ok := noMBScr.GetTxHandler().(*smartContractResult.SmartContractResult) if !ok { - log.Warn("smartContractResultsProcessor.processSCRsFromMiniblock cannot convert TransactionHandler to scr", - "scr hash", hex.EncodeToString([]byte(txHandlerHash)), - ) continue } - scrs[txHandlerHash] = scr + indexerScr := proc.prepareSmartContractResult([]byte(scrHash), nil, scr, header, selfShardID, selfShardID, noMBScr.GetFee(), noMBScr.GetGasUsed()) + + allSCRs = append(allSCRs, indexerScr) } - return scrs + return allSCRs } func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( header coreData.HeaderHandler, mb *block.MiniBlock, - scrs map[string]*smartContractResult.SmartContractResult, + scrs map[string]data.TransactionHandlerWithGasUsedAndFee, ) []*indexerData.ScResult { mbHash, err := core.CalculateHash(proc.marshalizer, proc.hasher, mb) if err != nil { @@ -98,15 +86,19 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( indexerSCRs := make([]*indexerData.ScResult, 0, len(mb.TxHashes)) for _, scrHash := range mb.TxHashes { - scr, ok := scrs[string(scrHash)] + scrHandler, ok := scrs[string(scrHash)] if !ok { log.Warn("smartContractResultsProcessor.processSCRsFromMiniblock scr not found in map", "scr hash", hex.EncodeToString(scrHash), ) continue } + scr, ok := scrHandler.GetTxHandler().(*smartContractResult.SmartContractResult) + if !ok { + continue + } - indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scr, header, mb.SenderShardID, mb.ReceiverShardID) + indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scr, header, mb.SenderShardID, mb.ReceiverShardID, scrHandler.GetFee(), scrHandler.GetGasUsed()) indexerSCRs = append(indexerSCRs, indexerSCR) delete(scrs, string(scrHash)) @@ -122,6 +114,8 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( header coreData.HeaderHandler, senderShard uint32, receiverShard uint32, + initialTxFee *big.Int, + initialTxGasUsed uint64, ) *indexerData.ScResult { hexEncodedMBHash := "" if len(mbHash) > 0 { @@ -174,6 +168,8 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( ReceiversShardIDs: res.ReceiversShardID, IsRelayed: res.IsRelayed, OriginalSender: originalSenderAddr, + InitialTxFee: initialTxFee.String(), + InitialTxGasUsed: initialTxGasUsed, } } diff --git a/process/transactions/smartContractResultsProcessor_test.go b/process/transactions/smartContractResultsProcessor_test.go index 998392a0..13b016f0 100644 --- a/process/transactions/smartContractResultsProcessor_test.go +++ b/process/transactions/smartContractResultsProcessor_test.go @@ -2,6 +2,7 @@ package transactions import ( "encoding/hex" + "math/big" "testing" "time" @@ -48,7 +49,7 @@ func TestPrepareSmartContractResult(t *testing.T) { header := &block.Header{TimeStamp: 100} mbHash := []byte("hash") - scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, smartContractRes, header, 0, 1) + scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, smartContractRes, header, 0, 1, big.NewInt(0), 0) expectedTx := &data.ScResult{ Nonce: nonce, Hash: hex.EncodeToString([]byte(scHash)), @@ -66,6 +67,7 @@ func TestPrepareSmartContractResult(t *testing.T) { Operation: "transfer", SenderAddressBytes: sndAddr, Receivers: []string{}, + InitialTxFee: "0", } require.Equal(t, expectedTx, scRes) diff --git a/process/transactions/transactionDBBuilder.go b/process/transactions/transactionDBBuilder.go index 848d33c4..c0c543d3 100644 --- a/process/transactions/transactionDBBuilder.go +++ b/process/transactions/transactionDBBuilder.go @@ -3,6 +3,7 @@ package transactions import ( "encoding/hex" "fmt" + "math/big" "time" indexer "github.com/ElrondNetwork/elastic-indexer-go" @@ -21,20 +22,17 @@ const emptyString = "" type dbTransactionBuilder struct { addressPubkeyConverter core.PubkeyConverter shardCoordinator indexer.ShardCoordinator - txFeeCalculator indexer.FeesProcessorHandler dataFieldParser DataFieldParser } func newTransactionDBBuilder( addressPubkeyConverter core.PubkeyConverter, shardCoordinator indexer.ShardCoordinator, - txFeeCalculator indexer.FeesProcessorHandler, dataFieldParser DataFieldParser, ) *dbTransactionBuilder { return &dbTransactionBuilder{ addressPubkeyConverter: addressPubkeyConverter, shardCoordinator: shardCoordinator, - txFeeCalculator: txFeeCalculator, dataFieldParser: dataFieldParser, } } @@ -46,10 +44,9 @@ func (dtb *dbTransactionBuilder) prepareTransaction( mb *block.MiniBlock, header coreData.HeaderHandler, txStatus string, + fee *big.Int, + gasUsed uint64, ) *data.Transaction { - gasUsed := dtb.txFeeCalculator.ComputeGasLimit(tx) - fee := dtb.txFeeCalculator.ComputeTxFeeBasedOnGasUsed(tx, gasUsed) - isScCall := core.IsSmartContractAddress(tx.RcvAddr) res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr) diff --git a/process/transactions/transactionDBBuilder_test.go b/process/transactions/transactionDBBuilder_test.go index 5160c66a..7ddc447d 100644 --- a/process/transactions/transactionDBBuilder_test.go +++ b/process/transactions/transactionDBBuilder_test.go @@ -10,7 +10,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elrond-go-core/core" - coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" "github.com/ElrondNetwork/elrond-go-core/data/transaction" @@ -20,16 +19,8 @@ import ( func createCommonProcessor() dbTransactionBuilder { return dbTransactionBuilder{ addressPubkeyConverter: mock.NewPubkeyConverterMock(32), - txFeeCalculator: &mock.EconomicsHandlerStub{ - ComputeTxFeeBasedOnGasUsedCalled: func(tx coreData.TransactionWithFeeHandler, gasUsed uint64) *big.Int { - return big.NewInt(100) - }, - ComputeGasLimitCalled: func(tx coreData.TransactionWithFeeHandler) uint64 { - return 500 - }, - }, - shardCoordinator: &mock.ShardCoordinatorMock{}, - dataFieldParser: createDataFieldParserMock(), + shardCoordinator: &mock.ShardCoordinatorMock{}, + dataFieldParser: createDataFieldParserMock(), } } @@ -86,7 +77,7 @@ func TestGetMoveBalanceTransaction(t *testing.T) { Receivers: []string{}, } - dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status) + dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500) require.Equal(t, expectedTx, dbTx) } diff --git a/process/transactions/transactionsGrouper.go b/process/transactions/transactionsGrouper.go index bcb79e71..25d45761 100644 --- a/process/transactions/transactionsGrouper.go +++ b/process/transactions/transactionsGrouper.go @@ -110,7 +110,7 @@ func (tg *txsGrouper) prepareNormalTxForDB( return nil, false } - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, mbStatus) + dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, mbStatus, txHandler.GetFee(), txHandler.GetGasUsed()) return dbTx, true } @@ -212,11 +212,7 @@ func (tg *txsGrouper) prepareInvalidTxForDB( return nil, false } - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String()) - - dbTx.GasUsed = dbTx.GasLimit - fee := tg.txBuilder.txFeeCalculator.ComputeTxFeeBasedOnGasUsed(tx, dbTx.GasUsed) - dbTx.Fee = fee.String() + dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), txHandler.GetFee(), txHandler.GetGasUsed()) return dbTx, true } diff --git a/process/transactions/transactionsGrouper_test.go b/process/transactions/transactionsGrouper_test.go index 975de455..a8d91dff 100644 --- a/process/transactions/transactionsGrouper_test.go +++ b/process/transactions/transactionsGrouper_test.go @@ -19,7 +19,7 @@ func TestGroupNormalTxs(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, &mock.EconomicsHandlerStub{}, parser) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -50,7 +50,7 @@ func TestGroupRewardsTxs(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, &mock.EconomicsHandlerStub{}, parser) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -79,7 +79,7 @@ func TestGroupInvalidTxs(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), &mock.ShardCoordinatorMock{}, &mock.EconomicsHandlerStub{}, parser) + txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), &mock.ShardCoordinatorMock{}, parser) grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -110,7 +110,7 @@ func TestGroupReceipts(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, &mock.EconomicsHandlerStub{}, parser) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") diff --git a/process/transactions/transactionsProcessor.go b/process/transactions/transactionsProcessor.go index cda3504c..2d5f0c1e 100644 --- a/process/transactions/transactionsProcessor.go +++ b/process/transactions/transactionsProcessor.go @@ -22,7 +22,6 @@ var log = logger.GetOrCreate("indexer/process/transactions") // new instances type ArgsTransactionProcessor struct { AddressPubkeyConverter core.PubkeyConverter - TxFeeCalculator indexer.FeesProcessorHandler ShardCoordinator indexer.ShardCoordinator Hasher hashing.Hasher Marshalizer marshal.Marshalizer @@ -30,11 +29,10 @@ type ArgsTransactionProcessor struct { } type txsDatabaseProcessor struct { - txFeeCalculator indexer.FeesProcessorHandler - txBuilder *dbTransactionBuilder - txsGrouper *txsGrouper - scrsProc *smartContractResultsProcessor - scrsDataToTxs *scrsDataToTransactions + txBuilder *dbTransactionBuilder + txsGrouper *txsGrouper + scrsProc *smartContractResultsProcessor + scrsDataToTxs *scrsDataToTransactions } // NewTransactionsProcessor will create a new instance of transactions database processor @@ -55,10 +53,10 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce } selfShardID := args.ShardCoordinator.SelfId() - txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, args.ShardCoordinator, args.TxFeeCalculator, operationsDataParser) + txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, args.ShardCoordinator, operationsDataParser) txsDBGrouper := newTxsGrouper(txBuilder, args.IsInImportMode, selfShardID, args.Hasher, args.Marshalizer) scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.ShardCoordinator, args.Marshalizer, args.Hasher, operationsDataParser) - scrsDataToTxs := newScrsDataToTransactions(args.TxFeeCalculator) + scrsDataToTxs := newScrsDataToTransactions() if args.IsInImportMode { log.Warn("the node is in import mode! Cross shard transactions and rewards where destination shard is " + @@ -66,11 +64,10 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce } return &txsDatabaseProcessor{ - txFeeCalculator: args.TxFeeCalculator, - txBuilder: txBuilder, - txsGrouper: txsDBGrouper, - scrsProc: scrProc, - scrsDataToTxs: scrsDataToTxs, + txBuilder: txBuilder, + txsGrouper: txsDBGrouper, + scrsProc: scrProc, + scrsDataToTxs: scrsDataToTxs, }, nil } @@ -136,7 +133,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( srcsNoTxInCurrentShard := tdp.scrsDataToTxs.attachSCRsToTransactionsAndReturnSCRsWithoutTx(normalTxs, dbSCResults) tdp.scrsDataToTxs.processTransactionsAfterSCRsWereAttached(normalTxs) - txHashStatus, txHashRefund := tdp.scrsDataToTxs.processSCRsWithoutTx(srcsNoTxInCurrentShard) + txHashStatus, txHashFee := tdp.scrsDataToTxs.processSCRsWithoutTx(srcsNoTxInCurrentShard) sliceNormalTxs := convertMapTxsToSlice(normalTxs) sliceRewardsTxs := convertMapTxsToSlice(rewardsTxs) @@ -148,7 +145,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( Receipts: dbReceipts, AlteredAccts: alteredAccounts, TxHashStatus: txHashStatus, - TxHashRefund: txHashRefund, + TxHashFee: txHashFee, } } diff --git a/process/transactions/transactionsProcessor_test.go b/process/transactions/transactionsProcessor_test.go index c213d82a..18bf963c 100644 --- a/process/transactions/transactionsProcessor_test.go +++ b/process/transactions/transactionsProcessor_test.go @@ -24,7 +24,6 @@ import ( func createMockArgsTxsDBProc() *ArgsTransactionProcessor { args := &ArgsTransactionProcessor{ AddressPubkeyConverter: mock.NewPubkeyConverterMock(10), - TxFeeCalculator: &mock.EconomicsHandlerStub{}, ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, @@ -468,7 +467,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 100, - }, 0, big.NewInt(0)) + }, 100, big.NewInt(0)) recHash1 := []byte("recHash1") rec1 := outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ Value: big.NewInt(100), @@ -558,7 +557,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * GasLimit: 100, GasPrice: 123456, Data: []byte("ESDTTransfer@54474e2d383862383366@0a"), - }, 0, big.NewInt(0)) + }, 100, big.NewInt(0)) scResHash1 := []byte("scResHash1") scRes1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, @@ -603,17 +602,13 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t t.Parallel() txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - txDbProc.scrsDataToTxs.txFeeCalculator = &mock.EconomicsHandlerStub{ - ComputeGasUsedAndFeeBasedOnRefundValueCalled: func(tx coreData.TransactionWithFeeHandler, refundValue *big.Int) (uint64, *big.Int) { - return 100, big.NewInt(10000) - }} txHash1 := []byte("txHash1") tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ GasLimit: 100, GasPrice: 123456, Data: []byte("ESDTNFTTransfer@595959453643392D303837363661@01@01@000000000000000005005C83E0C42EDCE394F40B24D29D298B0249C41F028974@66756E64@890479AFC610F4BEBC087D3ADA3F7C2775C736BBA91F41FD3D65092AA482D8B0@1c20"), - }, 0, big.NewInt(0)) + }, 100, big.NewInt(0)) scResHash1 := []byte("scResHash1") scRes1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 1, From 0675d2248c33fa02dede410bb96573f51bdb8675 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 12 Aug 2022 17:35:31 +0300 Subject: [PATCH 24/96] make tests compile --- .../accountsBalanceNftTransfer_test.go | 19 ++--- .../accountsBalanceWithLowerTimestamp_test.go | 27 +++--- .../accountsESDTWithTokenType_test.go | 33 ++++---- integrationtests/claimRewards_test.go | 20 ++--- integrationtests/collectionsIndex_test.go | 7 +- integrationtests/createNFTWithTags_test.go | 5 +- integrationtests/esdtTransfer_test.go | 16 ++-- .../issueTokenAndSetRoles_test.go | 28 +++---- integrationtests/issueToken_test.go | 8 +- integrationtests/nftIssueCreateBurn_test.go | 13 ++- .../nftTransferCrossShard_test.go | 84 +++++++++---------- .../nftTransferCrossWithScCall_test.go | 16 ++-- integrationtests/relayedTx_test.go | 39 +++++---- integrationtests/scCallIntraShard_test.go | 35 ++++---- integrationtests/transactions_test.go | 12 +-- integrationtests/updateNFT_test.go | 14 ++-- 16 files changed, 180 insertions(+), 196 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 12be3645..a35c974b 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -12,6 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -22,8 +23,6 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} - // ################ CREATE NFT ########################## shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: 0, @@ -34,7 +33,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { addr := "test-address-balance-1" addrHex := hex.EncodeToString([]byte(addr)) - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header := &dataBlock.Header{ @@ -42,7 +41,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { TimeStamp: 5600, } - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -60,10 +59,10 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + coreAlteredAccounts := map[string]*outport.AlteredAccount{ addrHex: { Address: addrHex, - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "NFT-abcdef", Nonce: 7440483, @@ -89,7 +88,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { TimeStamp: 5600, } - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -110,16 +109,16 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { addrReceiver := "new-address" addrReceiverHex := hex.EncodeToString([]byte(addrReceiver)) - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) - coreAlteredAccounts = map[string]*indexer.AlteredAccount{ + coreAlteredAccounts = map[string]*outport.AlteredAccount{ addrHex: { Address: addrHex, }, addrReceiverHex: { Address: addrReceiverHex, - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "NFT-abcdef", Nonce: 7440483, diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 9d723158..74327bec 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -13,6 +13,7 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -23,8 +24,6 @@ func TestIndexAccountsBalance(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} - // ################ UPDATE ACCOUNT-ESDT BALANCE ########################## shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: 0, @@ -41,10 +40,10 @@ func TestIndexAccountsBalance(t *testing.T) { encodedAddr := hex.EncodeToString([]byte(addr)) encodedAddr2 := hex.EncodeToString([]byte(addr2)) - alteredAccount := &indexer.AlteredAccount{ + alteredAccount := &outport.AlteredAccount{ Address: encodedAddr, Balance: "0", - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "TTTT-abcd", Balance: "1000", @@ -53,12 +52,12 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + coreAlteredAccounts := map[string]*outport.AlteredAccount{ encodedAddr: alteredAccount, encodedAddr2: alteredAccount, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header := &dataBlock.Header{ @@ -66,7 +65,7 @@ func TestIndexAccountsBalance(t *testing.T) { TimeStamp: 5600, } - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -106,7 +105,7 @@ func TestIndexAccountsBalance(t *testing.T) { TimeStamp: 5000, } - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -129,11 +128,11 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Balance = "2000" - pool = &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - "h1": &transaction.Transaction{ + pool = &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "h1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ SndAddr: []byte("eeeebbbb"), - }, + }, 0, big.NewInt(0)), }, Logs: []*coreData.LogData{ { @@ -179,7 +178,7 @@ func TestIndexAccountsBalance(t *testing.T) { esdtToken.Value = big.NewInt(0) encodedAddr = hex.EncodeToString([]byte(addr)) - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header = &dataBlock.Header{ @@ -190,7 +189,7 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Balance = "2000" coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" - pool.Txs = make(map[string]coreData.TransactionHandler) + pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index e2f11518..e7f409aa 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -15,6 +15,7 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -25,14 +26,12 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} - // ################ ISSUE NON FUNGIBLE TOKEN ########################## shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) body := &dataBlock.Body{} @@ -41,7 +40,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { TimeStamp: 5040, } - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -59,7 +58,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -75,11 +74,11 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { addr := "aaaabbbb" encodedAddr := hex.EncodeToString([]byte(addr)) - coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + coreAlteredAccounts := map[string]*outport.AlteredAccount{ encodedAddr: { Address: encodedAddr, Balance: "1000", - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "SEMI-abcd", Balance: "1000", @@ -92,7 +91,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, }, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header = &dataBlock.Header{ @@ -107,7 +106,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { } esdtDataBytes, _ := json.Marshal(esdtData) - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -142,8 +141,6 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} - // ################ CREATE SEMI FUNGIBLE TOKEN ########################## shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: 0, @@ -153,11 +150,11 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) addr := "aaaabbbb" encodedAddr := hex.EncodeToString([]byte(addr)) - coreAlteredAccounts := map[string]*indexer.AlteredAccount{ + coreAlteredAccounts := map[string]*outport.AlteredAccount{ encodedAddr: { Address: encodedAddr, Balance: "1000", - Tokens: []*indexer.AccountTokenData{ + Tokens: []*outport.AccountTokenData{ { Identifier: "TTTT-abcd", Nonce: 2, @@ -170,7 +167,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, }, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header := &dataBlock.Header{ @@ -185,7 +182,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) } esdtDataBytes, _ := json.Marshal(esdtData) - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -223,10 +220,10 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) TimeStamp: 5040, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -244,7 +241,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids = []string{"TTTT-abcd"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index bcfcd881..f4b9fe20 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -12,7 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" @@ -24,13 +24,11 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("claimRewards") @@ -91,13 +89,13 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { Value: big.NewInt(0), } - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): tx1, + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(tx1, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash2): scr2, - string(scrHash1): scr1, + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, Logs: []*coreData.LogData{ { @@ -115,7 +113,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, nil) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index caed2651..46499d85 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -30,8 +30,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { SelfID: core.MetachainShardId, } - feeComputer := &mock.EconomicsHandlerMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) body := &dataBlock.Body{} @@ -104,7 +103,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, }, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header = &dataBlock.Header{ @@ -195,7 +194,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) pool = &outport.Pool{ diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index d5561adb..29b1f806 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -34,7 +34,6 @@ func TestCreateNFTWithTags(t *testing.T) { }, } - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: 0, } @@ -47,7 +46,7 @@ func TestCreateNFTWithTags(t *testing.T) { addr := "aaaabbbb" addrHex := hex.EncodeToString([]byte(addr)) - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) esdtDataBytes, _ := json.Marshal(esdtToken) @@ -164,7 +163,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 3 coreAlteredAccounts[addrHex].Tokens[0].MetaData.Attributes = attributes - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) pool = &outport.Pool{ diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 518356f3..82620bd4 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -11,6 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" @@ -26,10 +27,9 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("esdtTransfer") @@ -87,13 +87,13 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { OriginalTxHash: txHash, } - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): txESDT, + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(txESDT, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash2): scr2, - string(scrHash1): scr1, + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index ab573de0..92fe4b7e 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -11,6 +11,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -21,12 +22,11 @@ func TestIssueTokenAndSetRole(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) body := &dataBlock.Body{} @@ -35,7 +35,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { TimeStamp: 5040, } - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -58,7 +58,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -68,7 +68,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-issue-ok.json"), string(genericResponse.Docs[0].Source)) // SET ROLES - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -97,7 +97,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) // TRANSFER ROLE - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -130,7 +130,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-transfer-role.json"), string(genericResponse.Docs[0].Source)) // UNSET ROLES - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -149,7 +149,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -165,13 +165,11 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} - shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) body := &dataBlock.Body{} @@ -181,7 +179,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } // SET ROLES - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -200,7 +198,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -210,7 +208,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) // ISSUE - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -228,7 +226,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*indexer.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index f20f27ce..0128ea14 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -11,6 +11,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -21,12 +22,11 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) body := &dataBlock.Body{} @@ -35,7 +35,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { TimeStamp: 5040, } - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -63,7 +63,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi.json"), string(genericResponse.Docs[0].Source)) // transfer ownership - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index eb8bef59..ddd01f4c 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -13,6 +13,7 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -23,14 +24,12 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} - // ################ ISSUE NON FUNGIBLE TOKEN ########################## shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: core.MetachainShardId, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) body := &dataBlock.Body{} @@ -39,7 +38,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { TimeStamp: 5040, } - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -71,7 +70,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { SelfID: 0, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err = CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header = &dataBlock.Header{ @@ -86,7 +85,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { } esdtDataBytes, _ := json.Marshal(esdtData) - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -120,7 +119,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { TimeStamp: 5666, } - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 278aefbe..ccd336ee 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -4,6 +4,7 @@ package integrationtests import ( "encoding/hex" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -22,10 +23,9 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("nftTransferWithSCCall") @@ -63,9 +63,9 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { OriginalTxHash: txHash, } - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): &transaction.Transaction{ + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: 79, SndAddr: []byte("erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50"), RcvAddr: []byte("erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50"), @@ -73,10 +73,10 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@4c4b4641524d2d336634663962@016534@6f1e6f01bc7627f5ae@00000000000000000500f1c8f2fdc58a63c6b201fc2ed629962d3dfa33fe7ceb@636f6d706f756e645265776172647350726f7879@000000000000000005004f79ec44bb13372b5ac9d996d749120f476427627ceb"), Value: big.NewInt(0), - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): scr1, + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) @@ -105,17 +105,17 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { } scrWithRefund := []byte("scrWithRefund") refundValueBig, _ := big.NewInt(0).SetString("101676480000000", 10) - poolDstShard := &indexer.Pool{ - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): scr1, - string(scrWithRefund): &smartContractResult.SmartContractResult{ + poolDstShard := &outport.Pool{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + string(scrWithRefund): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ SndAddr: []byte("erd1qqqqqqqqqqqqqpgq57szwud2quysucrlq2e97ntdysdl7v4ejz3qn3njq4"), RcvAddr: []byte("erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50"), PrevTxHash: []byte("f639cb7a0231191e04ec19dcb1359bd93a03fe8dc4a28a80d00835c5d1c988f8"), OriginalTxHash: txHash, Value: refundValueBig, Data: []byte("@6f6b@017d15@0000000e4d45584641524d2d6239336536300000000000017d15000000097045173cc97554b65d@0178af"), - }, + }, 0, big.NewInt(0)), }, } @@ -146,10 +146,9 @@ func TestNFTTransferCrossShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("nftTransfer") @@ -187,9 +186,9 @@ func TestNFTTransferCrossShard(t *testing.T) { } refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): &transaction.Transaction{ + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: 79, SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), @@ -197,10 +196,10 @@ func TestNFTTransferCrossShard(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@536f6d657468696e672d616263646566@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), Value: big.NewInt(0), - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): &smartContractResult.SmartContractResult{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 80, Value: refundValueBig, GasPrice: 1000000000, @@ -209,8 +208,8 @@ func TestNFTTransferCrossShard(t *testing.T) { Data: []byte("@6f6b"), PrevTxHash: txHash, OriginalTxHash: txHash, - }, - string(scrHash2): scr2, + }, 0, big.NewInt(0)), + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) @@ -238,16 +237,16 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } scr3WithErrHash := []byte("scrWithError") - poolDstShard := &indexer.Pool{ - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash2): scr2, - string(scr3WithErrHash): &smartContractResult.SmartContractResult{ + poolDstShard := &outport.Pool{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + string(scr3WithErrHash): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ SndAddr: []byte("erd1qqqqqqqqqqqqqpgq57szwud2quysucrlq2e97ntdysdl7v4ejz3qn3njq4"), RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), PrevTxHash: []byte("1546eb9970a6dc1710b6528274e75d5095c1349706f4ff70f52a1f58e1156316"), OriginalTxHash: txHash, Data: []byte("ESDTNFTTransfer@434f4c45435449452d323663313838@01@01@08011202000122e50108011204434f4f4c1a20e0f3ecf555f63f2d101241dfc98b4614aff9284edd50b46a1c6e36b83558744d20c4132a2e516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a324368747470733a2f2f697066732e696f2f697066732f516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a3a41746167733a436f6f6c3b6d657461646174613a516d5869417850396e535948515954546143357358717a4d32645856334142516145355241725932777a4e686179@75736572206572726f72"), - }, + }, 0, big.NewInt(0)), }, } @@ -269,10 +268,9 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("nftTransferCross") @@ -305,16 +303,16 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t }, } scr3WithErrHash := []byte("scrWithError") - poolDstShard := &indexer.Pool{ - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash2): scr2, - string(scr3WithErrHash): &smartContractResult.SmartContractResult{ + poolDstShard := &outport.Pool{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + string(scr3WithErrHash): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ SndAddr: []byte("erd1qqqqqqqqqqqqqpgq57szwud2quysucrlq2e97ntdysdl7v4ejz3qn3njq4"), RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), PrevTxHash: []byte("1546eb9970a6dc1710b6528274e75d5095c1349706f4ff70f52a1f58e1156316"), OriginalTxHash: txHash, Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@08011202000122e50108011204434f4f4c1a20e0f3ecf555f63f2d101241dfc98b4614aff9284edd50b46a1c6e36b83558744d20c4132a2e516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a324368747470733a2f2f697066732e696f2f697066732f516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a3a41746167733a436f6f6c3b6d657461646174613a516d5869417850396e535948515954546143357358717a4d32645856334142516145355241725932777a4e686179@75736572206572726f72"), - }, + }, 0, big.NewInt(0)), }, } @@ -351,9 +349,9 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t } refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): &transaction.Transaction{ + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: 79, SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), @@ -361,10 +359,10 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), Value: big.NewInt(0), - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): &smartContractResult.SmartContractResult{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 80, Value: refundValueBig, GasPrice: 1000000000, @@ -373,8 +371,8 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t Data: []byte("@6f6b"), PrevTxHash: txHash, OriginalTxHash: txHash, - }, - string(scrHash2): scr2, + }, 0, big.NewInt(0)), + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index e23da30c..13adb271 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -4,6 +4,7 @@ package integrationtests import ( "encoding/hex" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -26,10 +27,9 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("nftTransferWithScCall") @@ -66,9 +66,9 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { } // refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): &transaction.Transaction{ + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: 79, SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), @@ -76,10 +76,10 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@4d45584641524d2d636362323532@078b@0347543e5b59c9be8670@0801120b000347543e5b59c9be86702266088b0f1a20000000000000000005005754e4f6ba0b94efd71a0e4dd4814ee24e5f75297ceb32003a3d0000000701b6408636587c0000000000000410000000000000041001000000000a0347543e5b59c9be8670000000000000000a0347543e5b59c9be8670@636c61696d52657761726473"), Value: big.NewInt(0), - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash2): scr2, + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index a006f48d..4047acbf 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -4,6 +4,7 @@ package integrationtests import ( "encoding/hex" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -29,10 +30,9 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("relayedTx") @@ -80,12 +80,12 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { OriginalTxHash: txHash, } - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): initialTx, + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): scr1, + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) @@ -111,10 +111,10 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { } scrWithRefund := []byte("scrWithRefund") refundValueBig, _ := big.NewInt(0).SetString("86271830000000", 10) - poolDstShard := &indexer.Pool{ - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): scr1, - string(scrWithRefund): &smartContractResult.SmartContractResult{ + poolDstShard := &outport.Pool{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + string(scrWithRefund): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 3, SndAddr: []byte("erd1qqqqqqqqqqqqqpgq3dswlnnlkfd3gqrcv3dhzgnvh8ryf27g5rfsecnn2s"), RcvAddr: []byte("erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u"), @@ -123,7 +123,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { Value: refundValueBig, Data: []byte(""), ReturnMessage: []byte("gas refund for relayer"), - }, + }, 0, big.NewInt(0)), }, } @@ -142,10 +142,9 @@ func TestRelayedTransactionIntraShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("relayedTxIntra") @@ -205,13 +204,13 @@ func TestRelayedTransactionIntraShard(t *testing.T) { Value: refundValueBig, } - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): initialTx, + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): scr1, - string(scrHash2): scr2, + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 052e7bc6..21c63289 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -4,6 +4,7 @@ package integrationtests import ( "encoding/hex" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -27,10 +28,9 @@ func TestTransactionWithSCCallFail(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("t") @@ -51,9 +51,9 @@ func TestTransactionWithSCCallFail(t *testing.T) { } refundValueBig, _ := big.NewInt(0).SetString("5000000000000000000", 10) - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): &transaction.Transaction{ + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: 46, SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), RcvAddr: []byte("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqfhllllscrt56r"), @@ -61,10 +61,10 @@ func TestTransactionWithSCCallFail(t *testing.T) { GasPrice: 1000000000, Data: []byte("delegate"), Value: refundValueBig, - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): &smartContractResult.SmartContractResult{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 46, Value: refundValueBig, GasPrice: 0, @@ -74,7 +74,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { PrevTxHash: txHash, OriginalTxHash: txHash, ReturnMessage: []byte("total delegation cap reached"), - }, + }, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) @@ -94,10 +94,9 @@ func TestTransactionWithScCallSuccess(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("txHashClaimRewards") @@ -118,9 +117,9 @@ func TestTransactionWithScCallSuccess(t *testing.T) { } refundValueBig, _ := big.NewInt(0).SetString("2161082850000000", 10) - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): &transaction.Transaction{ + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: 101, SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), RcvAddr: []byte("erd1qqqqqqqqqqqqqpgqxwakt2g7u9atsnr03gqcgmhcv38pt7mkd94q6shuwt"), @@ -128,10 +127,10 @@ func TestTransactionWithScCallSuccess(t *testing.T) { GasPrice: 1000000000, Data: []byte("claimRewards"), Value: big.NewInt(0), - }, + }, 0, big.NewInt(0)), }, - Scrs: map[string]coreData.TransactionHandler{ - string(scrHash1): &smartContractResult.SmartContractResult{ + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ Nonce: 102, Value: refundValueBig, GasPrice: 1000000000, @@ -140,7 +139,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { Data: []byte("@6f6b"), PrevTxHash: txHash, OriginalTxHash: txHash, - }, + }, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index ec1ff15a..2097da33 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -11,6 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -23,10 +24,9 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) txHash := []byte("hash") @@ -44,9 +44,9 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { }, }, } - pool := &indexer.Pool{ - Txs: map[string]coreData.TransactionHandler{ - string(txHash): &transaction.Transaction{ + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ Nonce: 1, SndAddr: []byte("sender"), RcvAddr: []byte("receiver"), @@ -54,7 +54,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { GasPrice: 1000000000, Data: []byte("transfer"), Value: big.NewInt(1234), - }, + }, 0, big.NewInt(0)), }, } err = esProc.SaveTransactions(body, header, pool, nil) diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index acaba366..73be4843 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -13,6 +13,7 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -30,10 +31,9 @@ func TestNFTUpdateMetadata(t *testing.T) { } marshalizedCreate, _ := json.Marshal(esdtCreateData) - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header := &dataBlock.Header{ @@ -43,7 +43,7 @@ func TestNFTUpdateMetadata(t *testing.T) { body := &dataBlock.Body{} // CREATE NFT data - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ @@ -70,7 +70,7 @@ func TestNFTUpdateMetadata(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token.json"), string(genericResponse.Docs[0].Source)) // Add URIS 1 - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ @@ -91,7 +91,7 @@ func TestNFTUpdateMetadata(t *testing.T) { require.Nil(t, err) // Add URIS 2 --- results should be the same - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ @@ -118,7 +118,7 @@ func TestNFTUpdateMetadata(t *testing.T) { require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-add-uris.json"), string(genericResponse.Docs[0].Source)) - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ @@ -146,7 +146,7 @@ func TestNFTUpdateMetadata(t *testing.T) { // Update attributes 2 - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ From b35bc12de48fff77caaf73ec1df7284aee9a9d2d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 12 Aug 2022 17:36:45 +0300 Subject: [PATCH 25/96] fix imports --- integrationtests/nftTransferCrossShard_test.go | 2 +- integrationtests/nftTransferCrossWithScCall_test.go | 2 +- integrationtests/relayedTx_test.go | 2 +- integrationtests/scCallIntraShard_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index ccd336ee..ea90628a 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -4,7 +4,6 @@ package integrationtests import ( "encoding/hex" - "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -12,6 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 13adb271..a2709b2c 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -4,7 +4,6 @@ package integrationtests import ( "encoding/hex" - "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -12,6 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 4047acbf..76306762 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -4,7 +4,6 @@ package integrationtests import ( "encoding/hex" - "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -12,6 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 21c63289..fd2774e9 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -4,7 +4,6 @@ package integrationtests import ( "encoding/hex" - "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -12,6 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" From 5bc6d191f61a5488cc0e7f1b45cdcd5430b566e1 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 16 Aug 2022 10:53:39 +0300 Subject: [PATCH 26/96] fix integration tests --- integrationtests/claimRewards_test.go | 2 +- integrationtests/esdtTransfer_test.go | 8 ++--- .../nftTransferCrossShard_test.go | 8 ++--- .../nftTransferCrossWithScCall_test.go | 12 +++---- integrationtests/relayedTx_test.go | 28 ++++++++-------- integrationtests/scCallIntraShard_test.go | 19 ++++++----- .../testdata/esdtTransfer/esdt-transfer.json | 27 +++++++++++++++ .../cross-shard-transfer-with-sc-call.json | 33 +++++++++++++++++++ .../relayedTx/relayed-tx-after-refund.json | 29 ++++++++++++++++ .../testdata/relayedTx/relayed-tx-intra.json | 28 ++++++++++++++++ .../testdata/relayedTx/relayed-tx-source.json | 29 ++++++++++++++++ .../scCallIntraShard/claim-rewards.json | 21 ++++++++++++ .../scCallIntraShard/sc-call-fail.json | 21 ++++++++++++ .../testdata/transactions/move-balance.json | 20 +++++++++++ integrationtests/transactions_test.go | 9 ++--- 15 files changed, 250 insertions(+), 44 deletions(-) create mode 100644 integrationtests/testdata/esdtTransfer/esdt-transfer.json create mode 100644 integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json create mode 100644 integrationtests/testdata/relayedTx/relayed-tx-after-refund.json create mode 100644 integrationtests/testdata/relayedTx/relayed-tx-intra.json create mode 100644 integrationtests/testdata/relayedTx/relayed-tx-source.json create mode 100644 integrationtests/testdata/scCallIntraShard/claim-rewards.json create mode 100644 integrationtests/testdata/scCallIntraShard/sc-call-fail.json create mode 100644 integrationtests/testdata/transactions/move-balance.json diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index f4b9fe20..ac40d695 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -91,7 +91,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(tx1, 0, big.NewInt(0)), + string(txHash): outport.NewTransactionHandlerWithGasAndFee(tx1, 1068000, big.NewInt(78000000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 82620bd4..4c115ceb 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -17,10 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -const ( - expectedESDTTransferTX = `{"miniBlockHash":"1ecea6dff9ab9a785a2d55720e88c1bbd7d9c56310a035d16163e373879cd0e1","nonce":6,"round":50,"value":"0","receiver":"657264313375377a79656b7a7664767a656b38373638723567617539703636373775667070736a756b6c7539653674377978377268673473363865327a65","sender":"65726431656636343730746a64746c67706139663667336165346e7365646d6a6730677636773733763332787476686b6666663939336871373530786c39","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":104011,"gasUsed":104011,"fee":"104000110000000","data":"RVNEVFRyYW5zZmVyQDU0NDc0ZTJkMzgzODYyMzgzMzY2QDBh","signature":"","timestamp":5040,"status":"success","searchOrder":0,"hasScResults":true,"tokens":["TGN-88b83f"],"esdtValues":["10"],"operation":"ESDTTransfer"}` -) - func TestESDTTransferTooMuchGasProvided(t *testing.T) { setLogLevelDebug() @@ -89,7 +85,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(txESDT, 0, big.NewInt(0)), + string(txHash): outport.NewTransactionHandlerWithGasAndFee(txESDT, 104011, big.NewInt(104000110000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), @@ -104,5 +100,5 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) - compareTxs(t, []byte(expectedESDTTransferTX), genericResponse.Docs[0].Source) + require.JSONEq(t, readExpectedResult("./testdata/esdtTransfer/esdt-transfer.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index ea90628a..c38400a2 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -73,7 +73,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@4c4b4641524d2d336634663962@016534@6f1e6f01bc7627f5ae@00000000000000000500f1c8f2fdc58a63c6b201fc2ed629962d3dfa33fe7ceb@636f6d706f756e645265776172647350726f7879@000000000000000005004f79ec44bb13372b5ac9d996d749120f476427627ceb"), Value: big.NewInt(0), - }, 0, big.NewInt(0)), + }, 150000000, big.NewInt(1904415000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), @@ -115,7 +115,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { OriginalTxHash: txHash, Value: refundValueBig, Data: []byte("@6f6b@017d15@0000000e4d45584641524d2d6239336536300000000000017d15000000097045173cc97554b65d@0178af"), - }, 0, big.NewInt(0)), + }, 139832352, big.NewInt(1802738520000000)), }, } @@ -196,7 +196,7 @@ func TestNFTTransferCrossShard(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@536f6d657468696e672d616263646566@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), Value: big.NewInt(0), - }, 0, big.NewInt(0)), + }, 963500, big.NewInt(235850000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ @@ -359,7 +359,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), Value: big.NewInt(0), - }, 0, big.NewInt(0)), + }, 963500, big.NewInt(238820000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index a2709b2c..0b87145f 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -17,10 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -const ( - expectedCrossShardTransferWithSCCall = `{"miniBlockHash":"99a07aab4f6722a1473b33bd7bb35e339c69339c400737b14a94ad8bceaa1734","nonce":79,"round":50,"value":"0","receiver":"65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376","sender":"65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":5000000,"gasUsed":5000000,"fee":"595490000000000","data":"RVNEVE5GVFRyYW5zZmVyQDRkNDU1ODQ2NDE1MjRkMmQ2MzYzNjIzMjM1MzJAMDc4YkAwMzQ3NTQzZTViNTljOWJlODY3MEAwODAxMTIwYjAwMDM0NzU0M2U1YjU5YzliZTg2NzAyMjY2MDg4YjBmMWEyMDAwMDAwMDAwMDAwMDAwMDAwNTAwNTc1NGU0ZjZiYTBiOTRlZmQ3MWEwZTRkZDQ4MTRlZTI0ZTVmNzUyOTdjZWIzMjAwM2EzZDAwMDAwMDA3MDFiNjQwODYzNjU4N2MwMDAwMDAwMDAwMDAwNDEwMDAwMDAwMDAwMDAwMDQxMDAxMDAwMDAwMDAwYTAzNDc1NDNlNWI1OWM5YmU4NjcwMDAwMDAwMDAwMDAwMDAwYTAzNDc1NDNlNWI1OWM5YmU4NjcwQDYzNmM2MTY5NmQ1MjY1Nzc2MTcyNjQ3Mw==","signature":"","timestamp":5040,"status":"success","searchOrder":0,"hasScResults":true,"tokens":["MEXFARM-ccb252-078b"],"esdtValues":["15482888667631250736752"],"receivers":["0801120b000347543e5b59c9be86702266088b0f1a20000000000000000005005754e4f6ba0b94efd71a0e4dd4814ee24e5f75297ceb32003a3d0000000701b6408636587c0000000000000410000000000000041001000000000a0347543e5b59c9be8670000000000000000a0347543e5b59c9be8670"],"receiversShardIDs":[0],"operation":"ESDTNFTTransfer"}` -) - func TestNFTTransferCrossShardWithScCall(t *testing.T) { setLogLevelDebug() @@ -76,7 +72,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@4d45584641524d2d636362323532@078b@0347543e5b59c9be8670@0801120b000347543e5b59c9be86702266088b0f1a20000000000000000005005754e4f6ba0b94efd71a0e4dd4814ee24e5f75297ceb32003a3d0000000701b6408636587c0000000000000410000000000000041001000000000a0347543e5b59c9be8670000000000000000a0347543e5b59c9be8670@636c61696d52657761726473"), Value: big.NewInt(0), - }, 0, big.NewInt(0)), + }, 5000000, big.NewInt(595490000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), @@ -89,6 +85,8 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { genericResponse := &GenericResponse{} err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) - - compareTxs(t, []byte(expectedCrossShardTransferWithSCCall), genericResponse.Docs[0].Source) + require.JSONEq(t, + readExpectedResult("./testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json"), + string(genericResponse.Docs[0].Source), + ) } diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 76306762..ac037c34 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -17,13 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -const ( - expectedRelayedTxSource = `{"miniBlockHash":"fed7c174a849c30b88c36a26453407f1b95970941d0872e603e641c5c804104a","nonce":1196667,"round":50,"value":"0","receiver":"6572643134657961796672766c72687a66727767357a776c65756132356d6b7a676e6367676e33356e766336786876357978776d6c326573306633646874","sender":"657264316b376a3665776a736c61347a73677638763666366665336476726b677633643064396a6572637a773435687a6564687965643873683275333475","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":16610000,"gasUsed":16610000,"fee":"1760000000000000","data":"cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMyMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjQxNDE0MTQxNDE0MTQxNDE0MTQxNDE0NjQxNDk3NDY3MzczODM1MmY3MzZjNzM1NTQxNDg2ODZiNTczMzQ1Njk2MjRjNmU0NzUyNGI3NjQ5NmY0ZTRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzUzMDMwMzAzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI2MzMyNDYzMjVhNTU0NjMwNjQ0NzU2N2E2NDQ3NDYzMDYxNTczOTc1NTE0NDQ2Njg1OTdhNDkzMTRkNmE1OTM1NTk2ZDUxMzM1YTQ0NDk3NzU5MzI0YTY5NTk1NDRkMzE1OTZkNTY2YzRmNDQ1OTMxNGQ0NDY0Njg0ZjU3NGU2YTRlN2E2NzdhNWE0NzU1Nzc0ZjQ0NWE2OTRlNDQ0NTMzNGU1NDZiMzQ1YTU0NTE3YTU5NTQ0ZTZiNWE2YTU2NmE1OTMyNDU3OTVhNTQ2ODY4NGQ2YTZjNDE0ZDZhNTEzNDRlNTQ2NzdhNGQ1NzRlNmQ0ZDU0NDUzMDRkNTQ1NjZkNTk2YTQxMzU0ZDZhNjM3NzRlNDQ1MTMyNGU1NzU1MzI0ZTdhNTk3YTU5NTc0ZDMxNGY0NDQ1MzQ1YTU0NjczMTRlNDc1MTM0NTk1NzUyNmQ0ZTU0NDE3YTU5NmE2MzM1NGQ2YTZjNmI0ZjU0NTI2YzRlNmQ0OTc5NGU2YTQ5Nzc1YTY3M2QzZDIyMmMyMjYzNjg2MTY5NmU0OTQ0MjIzYTIyNGQ1MTNkM2QyMjJjMjI3NjY1NzI3MzY5NmY2ZTIyM2EzMTJjMjI3MzY5Njc2ZTYxNzQ3NTcyNjUyMjNhMjI1MjM5NDYyYjM0NTQ2MzUyNDE1YTM4NmQ3NzcxMzI0NTU5MzAzMTYzNTk2YzMzNzY2MjcxNmM0NjY1NzE3NjM4N2E3NjQ3NGE3NzVhNjgzMzU5NGQ0ZjU1NmI0MjM0NjQzNDUxNTc0ZTY2Mzc2NzQ0NjI2YzQ4NDgzMjU3NmI3MTYxNGE3NjYxNDg0NTc0NDM1NjYxNzA0OTcxMzM2NTM1NjU2MjM4NGU0MTc3M2QzZDIyN2Q=","signature":"","timestamp":5040,"status":"success","searchOrder":0,"hasScResults":true,"receivers":["000000000000000005008b60efce7fb25b140078645b71226cb9c644abc8a0d3"],"receiversShardIDs":[0],"operation":"transfer","function":"saveAttestation","isRelayed":true}` - expectedRelayedTxAfterRefund = `{"miniBlockHash":"fed7c174a849c30b88c36a26453407f1b95970941d0872e603e641c5c804104a","nonce":1196667,"round":50,"value":"0","receiver":"6572643134657961796672766c72687a66727767357a776c65756132356d6b7a676e6367676e33356e766336786876357978776d6c326573306633646874","sender":"657264316b376a3665776a736c61347a73677638763666366665336476726b677633643064396a6572637a773435687a6564687965643873683275333475","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":16610000,"gasUsed":7982817,"fee":"1673728170000000","data":"cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMyMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjQxNDE0MTQxNDE0MTQxNDE0MTQxNDE0NjQxNDk3NDY3MzczODM1MmY3MzZjNzM1NTQxNDg2ODZiNTczMzQ1Njk2MjRjNmU0NzUyNGI3NjQ5NmY0ZTRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzUzMDMwMzAzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI2MzMyNDYzMjVhNTU0NjMwNjQ0NzU2N2E2NDQ3NDYzMDYxNTczOTc1NTE0NDQ2Njg1OTdhNDkzMTRkNmE1OTM1NTk2ZDUxMzM1YTQ0NDk3NzU5MzI0YTY5NTk1NDRkMzE1OTZkNTY2YzRmNDQ1OTMxNGQ0NDY0Njg0ZjU3NGU2YTRlN2E2NzdhNWE0NzU1Nzc0ZjQ0NWE2OTRlNDQ0NTMzNGU1NDZiMzQ1YTU0NTE3YTU5NTQ0ZTZiNWE2YTU2NmE1OTMyNDU3OTVhNTQ2ODY4NGQ2YTZjNDE0ZDZhNTEzNDRlNTQ2NzdhNGQ1NzRlNmQ0ZDU0NDUzMDRkNTQ1NjZkNTk2YTQxMzU0ZDZhNjM3NzRlNDQ1MTMyNGU1NzU1MzI0ZTdhNTk3YTU5NTc0ZDMxNGY0NDQ1MzQ1YTU0NjczMTRlNDc1MTM0NTk1NzUyNmQ0ZTU0NDE3YTU5NmE2MzM1NGQ2YTZjNmI0ZjU0NTI2YzRlNmQ0OTc5NGU2YTQ5Nzc1YTY3M2QzZDIyMmMyMjYzNjg2MTY5NmU0OTQ0MjIzYTIyNGQ1MTNkM2QyMjJjMjI3NjY1NzI3MzY5NmY2ZTIyM2EzMTJjMjI3MzY5Njc2ZTYxNzQ3NTcyNjUyMjNhMjI1MjM5NDYyYjM0NTQ2MzUyNDE1YTM4NmQ3NzcxMzI0NTU5MzAzMTYzNTk2YzMzNzY2MjcxNmM0NjY1NzE3NjM4N2E3NjQ3NGE3NzVhNjgzMzU5NGQ0ZjU1NmI0MjM0NjQzNDUxNTc0ZTY2Mzc2NzQ0NjI2YzQ4NDgzMjU3NmI3MTYxNGE3NjYxNDg0NTc0NDM1NjYxNzA0OTcxMzM2NTM1NjU2MjM4NGU0MTc3M2QzZDIyN2Q=","signature":"","timestamp":5040,"status":"success","searchOrder":0,"hasScResults":true,"receivers":["000000000000000005008b60efce7fb25b140078645b71226cb9c644abc8a0d3"],"receiversShardIDs":[0],"operation":"transfer","function":"saveAttestation","isRelayed":true}` - - expectedRelayedTxIntra = `{"miniBlockHash":"2709174224d13e49fd76a70b48bd3db7838ca715bcfe09be59cef043241d7ef3","nonce":1196665,"round":50,"value":"0","receiver":"6572643134657961796672766c72687a66727767357a776c65756132356d6b7a676e6367676e33356e766336786876357978776d6c326573306633646874","sender":"657264316b376a3665776a736c61347a73677638763666366665336476726b677633643064396a6572637a773435687a6564687965643873683275333475","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":15406000,"gasUsed":10556000,"fee":"2257820000000000","data":"cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMwMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjcyNmI2ZTUzNGE0NzdhMzQzNzY5NTM0ZTc5NGI0MzY0MmY1MDRmNzE3MDc1Nzc2YjU0Nzc2ODQ1MzQzMDZkN2E0NzZhNTg1YTUxNjg2ZTYyMmI3MjRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzMzMjMzMzIzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI1NTMyNDYzMjVhNTU3NDZjNjU1NjVhNjg2MjQ4NTY2YzUxNDQ1OTc5NGU2YjU1MzI0ZDZiNDEzMjRkNmE1YTQ2NGU2YTQ5N2E0ZDU0NGQzNTRlNmE1NTMyNGQ1NDRkMzI0ZTZiNGQzMjUxNTQ2Mzc4NGQ3YTZiMzI1MTdhNjMzMDRlNmE1NTMzNGU0NDYzMzA0ZTdhNjczMjRlNTQ0ZDMzNGU3YTUxN2E0ZTdhNTkzMzRlNmI1NTdhNGQ0NDYzNzc0ZDdhNDk3YTRmNTQ2NDQyNGU3YTU5MzM0ZTU0NWE0NDRlNmE0NTMzNGU1NDU5MzI0ZTZhNjMzMzRlNTQ0ZDMwNGU3YTQ1N2E0ZTdhNTkzMzRlN2E0YTQxNGU2YTU1MzM0ZTQ0NTkzNDUxNDQ0ZDc3NGU3YTY3N2E0ZDU0NGQzMDRlNDQ1MTMyNGQ2YTRkNzg0ZTZhNTU3YTRkNDQ0ZDMxNGU2YTU5N2E0ZTU0NTE3YTRlNmE0NTdhNGU1NDRkMzE0ZTZhNDk3YTRkNTQ0ZDdhNGU0NDQ1MzA0ZDdhNTk3ODRkN2E2MzMyNGQ3YTRkMzA0ZTZhNGQ3YTRkNDQ0ZDc4NGU2YTU5N2E0ZTU0NTkzMDRlNDQ1NTdhNGU1NDRkMzE0ZTQ0NDk3YTRmNTQ1MTdhNGU0NDQ1MzI0ZTZhNTk3ODRkN2E2MzdhNGY1NTQxMzI0ZDZhNjMzMDRlNmE0ZTQxNGU2YTQ5MzI0ZDdhNGQ3ODRlN2E0NTMyNGQ1NDU5MzE0ZDdhNTU3YTRmNDQ0ZDdhNGQ3YTRkMzM0ZDZhNjM3OTRlN2E1OTdhNGU0NDRkNzc0ZDdhNDE3YTRlNTQ1YTQ0NGU2YTU5MzI1MjQ0NGQzMTRlN2E1OTMyNTI0NDRkMzI0ZTdhNTUzMjUxNTQ2MzMyNGU2YTUxN2E0ZTQ0NTk3YTRlN2E1MTdhNGY1NDVhNDI0ZTZhNGQ3YTRlNTQ1YTQyNGU3YTYzMzM1MTU0NGQzNDRlNmE1NTMzNGY0NDYzNzcyMjJjMjI2MzY4NjE2OTZlNDk0NDIyM2EyMjRkNTEzZDNkMjIyYzIyNzY2NTcyNzM2OTZmNmUyMjNhMzEyYzIyNzM2OTY3NmU2MTc0NzU3MjY1MjIzYTIyNzE2NjcwNGE0Nzc2NzM0NDQ0NDI1NTUxNGUyZjUyNTU0NzRmNTA1Mzc1NTIzMjQ4NGY0YTYxNGI3MDM4NDUzNjYzNGU1NDc3MzAzMzQzMzc2OTM0NTU3Nzc2MmY0YzU0NzM2ZDJiNmE3MDQyMzk3NTZjNDgzOTY2NTMyYjQ0NzE2MTcyNzE0ZjYyNDg0MTcwMzg2NjZkNzIzMDZhNDE1NTMxNzM2ZTM1NDE2NzNkM2QyMjdk","signature":"","timestamp":5040,"status":"success","searchOrder":0,"hasScResults":true,"receivers":["ae49d2246cf8ee248dc8a09dfcf3aaa6ec244f0844e349b31a35d94219dbfab3"],"receiversShardIDs":[0],"operation":"SaveKeyValue","isRelayed":true}` -) - func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { setLogLevelDebug() @@ -82,7 +75,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 0, big.NewInt(0)), + string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 16610000, big.NewInt(1760000000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), @@ -96,7 +89,10 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) - compareTxs(t, []byte(expectedRelayedTxSource), genericResponse.Docs[0].Source) + require.JSONEq(t, + readExpectedResult("./testdata/relayedTx/relayed-tx-source.json"), + string(genericResponse.Docs[0].Source), + ) // EXECUTE transfer on the destination shard bodyDstShard := &dataBlock.Body{ @@ -123,7 +119,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { Value: refundValueBig, Data: []byte(""), ReturnMessage: []byte("gas refund for relayer"), - }, 0, big.NewInt(0)), + }, 7982817, big.NewInt(1673728170000000)), }, } @@ -133,7 +129,10 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) - compareTxs(t, []byte(expectedRelayedTxAfterRefund), genericResponse.Docs[0].Source) + require.JSONEq(t, + readExpectedResult("./testdata/relayedTx/relayed-tx-after-refund.json"), + string(genericResponse.Docs[0].Source), + ) } func TestRelayedTransactionIntraShard(t *testing.T) { @@ -206,7 +205,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 0, big.NewInt(0)), + string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 10556000, big.NewInt(2257820000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), @@ -221,5 +220,8 @@ func TestRelayedTransactionIntraShard(t *testing.T) { err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) - require.JSONEq(t, expectedRelayedTxIntra, string(genericResponse.Docs[0].Source)) + require.JSONEq(t, + readExpectedResult("./testdata/relayedTx/relayed-tx-intra.json"), + string(genericResponse.Docs[0].Source), + ) } diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index fd2774e9..0ee770a9 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -17,11 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -const ( - claimRewardsTx = `{"miniBlockHash":"60b38b11110d28d1b361359f9688bb041bb9180219a612a83ff00dcc0db4d607","nonce":101,"round":50,"value":"0","receiver":"65726431717171717171717171717171717067717877616b7432673775396174736e723033677163676d68637633387074376d6b64393471367368757774","sender":"65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":250000000,"gasUsed":33891715,"fee":"406237150000000","data":"Y2xhaW1SZXdhcmRz","signature":"","timestamp":5040,"status":"success","searchOrder":0,"hasScResults":true,"operation":"transfer"}` - scCallFailTx = `{"miniBlockHash":"5d04f80b044352bfbbde123702323eae07fdd8ca77f24f256079006058b6e7b4","nonce":46,"round":50,"value":"5000000000000000000","receiver":"6572643171717171717171717171717171717170717171717171717171717171717171717171717171717171717171717166686c6c6c6c73637274353672","sender":"65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":12000000,"gasUsed":12000000,"fee":"181380000000000","data":"ZGVsZWdhdGU=","signature":"","timestamp":5040,"status":"fail","searchOrder":0,"hasScResults":true,"operation":"transfer"}` -) - func TestTransactionWithSCCallFail(t *testing.T) { setLogLevelDebug() @@ -61,7 +56,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { GasPrice: 1000000000, Data: []byte("delegate"), Value: refundValueBig, - }, 0, big.NewInt(0)), + }, 12000000, big.NewInt(181380000000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ @@ -85,7 +80,10 @@ func TestTransactionWithSCCallFail(t *testing.T) { err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) - compareTxs(t, []byte(scCallFailTx), genericResponse.Docs[0].Source) + require.JSONEq(t, + readExpectedResult("./testdata/scCallIntraShard/sc-call-fail.json"), + string(genericResponse.Docs[0].Source), + ) } func TestTransactionWithScCallSuccess(t *testing.T) { @@ -127,7 +125,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { GasPrice: 1000000000, Data: []byte("claimRewards"), Value: big.NewInt(0), - }, 0, big.NewInt(0)), + }, 33891715, big.NewInt(406237150000000)), }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ @@ -150,5 +148,8 @@ func TestTransactionWithScCallSuccess(t *testing.T) { err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) - compareTxs(t, []byte(claimRewardsTx), genericResponse.Docs[0].Source) + require.JSONEq(t, + readExpectedResult("./testdata/scCallIntraShard/claim-rewards.json"), + string(genericResponse.Docs[0].Source), + ) } diff --git a/integrationtests/testdata/esdtTransfer/esdt-transfer.json b/integrationtests/testdata/esdtTransfer/esdt-transfer.json new file mode 100644 index 00000000..1115aea8 --- /dev/null +++ b/integrationtests/testdata/esdtTransfer/esdt-transfer.json @@ -0,0 +1,27 @@ +{ + "miniBlockHash": "1ecea6dff9ab9a785a2d55720e88c1bbd7d9c56310a035d16163e373879cd0e1", + "nonce": 6, + "round": 50, + "value": "0", + "receiver": "657264313375377a79656b7a7664767a656b38373638723567617539703636373775667070736a756b6c7539653674377978377268673473363865327a65", + "sender": "65726431656636343730746a64746c67706139663667336165346e7365646d6a6730677636773733763332787476686b6666663939336871373530786c39", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 104011, + "gasUsed": 104011, + "fee": "104000110000000", + "data": "RVNEVFRyYW5zZmVyQDU0NDc0ZTJkMzgzODYyMzgzMzY2QDBh", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "hasScResults": true, + "tokens": [ + "TGN-88b83f" + ], + "esdtValues": [ + "10" + ], + "operation": "ESDTTransfer" +} diff --git a/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json b/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json new file mode 100644 index 00000000..cf498041 --- /dev/null +++ b/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json @@ -0,0 +1,33 @@ +{ + "miniBlockHash": "99a07aab4f6722a1473b33bd7bb35e339c69339c400737b14a94ad8bceaa1734", + "nonce": 79, + "round": 50, + "value": "0", + "receiver": "65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376", + "sender": "65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 5000000, + "gasUsed": 5000000, + "fee": "595490000000000", + "data": "RVNEVE5GVFRyYW5zZmVyQDRkNDU1ODQ2NDE1MjRkMmQ2MzYzNjIzMjM1MzJAMDc4YkAwMzQ3NTQzZTViNTljOWJlODY3MEAwODAxMTIwYjAwMDM0NzU0M2U1YjU5YzliZTg2NzAyMjY2MDg4YjBmMWEyMDAwMDAwMDAwMDAwMDAwMDAwNTAwNTc1NGU0ZjZiYTBiOTRlZmQ3MWEwZTRkZDQ4MTRlZTI0ZTVmNzUyOTdjZWIzMjAwM2EzZDAwMDAwMDA3MDFiNjQwODYzNjU4N2MwMDAwMDAwMDAwMDAwNDEwMDAwMDAwMDAwMDAwMDQxMDAxMDAwMDAwMDAwYTAzNDc1NDNlNWI1OWM5YmU4NjcwMDAwMDAwMDAwMDAwMDAwYTAzNDc1NDNlNWI1OWM5YmU4NjcwQDYzNmM2MTY5NmQ1MjY1Nzc2MTcyNjQ3Mw==", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "hasScResults": true, + "tokens": [ + "MEXFARM-ccb252-078b" + ], + "esdtValues": [ + "15482888667631250736752" + ], + "receivers": [ + "0801120b000347543e5b59c9be86702266088b0f1a20000000000000000005005754e4f6ba0b94efd71a0e4dd4814ee24e5f75297ceb32003a3d0000000701b6408636587c0000000000000410000000000000041001000000000a0347543e5b59c9be8670000000000000000a0347543e5b59c9be8670" + ], + "receiversShardIDs": [ + 0 + ], + "operation": "ESDTNFTTransfer" +} diff --git a/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json new file mode 100644 index 00000000..3df42223 --- /dev/null +++ b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json @@ -0,0 +1,29 @@ +{ + "miniBlockHash": "fed7c174a849c30b88c36a26453407f1b95970941d0872e603e641c5c804104a", + "nonce": 1196667, + "round": 50, + "value": "0", + "receiver": "6572643134657961796672766c72687a66727767357a776c65756132356d6b7a676e6367676e33356e766336786876357978776d6c326573306633646874", + "sender": "657264316b376a3665776a736c61347a73677638763666366665336476726b677633643064396a6572637a773435687a6564687965643873683275333475", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 16610000, + "gasUsed": 7982817, + "fee": "1673728170000000", + "data": "cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMyMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjQxNDE0MTQxNDE0MTQxNDE0MTQxNDE0NjQxNDk3NDY3MzczODM1MmY3MzZjNzM1NTQxNDg2ODZiNTczMzQ1Njk2MjRjNmU0NzUyNGI3NjQ5NmY0ZTRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzUzMDMwMzAzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI2MzMyNDYzMjVhNTU0NjMwNjQ0NzU2N2E2NDQ3NDYzMDYxNTczOTc1NTE0NDQ2Njg1OTdhNDkzMTRkNmE1OTM1NTk2ZDUxMzM1YTQ0NDk3NzU5MzI0YTY5NTk1NDRkMzE1OTZkNTY2YzRmNDQ1OTMxNGQ0NDY0Njg0ZjU3NGU2YTRlN2E2NzdhNWE0NzU1Nzc0ZjQ0NWE2OTRlNDQ0NTMzNGU1NDZiMzQ1YTU0NTE3YTU5NTQ0ZTZiNWE2YTU2NmE1OTMyNDU3OTVhNTQ2ODY4NGQ2YTZjNDE0ZDZhNTEzNDRlNTQ2NzdhNGQ1NzRlNmQ0ZDU0NDUzMDRkNTQ1NjZkNTk2YTQxMzU0ZDZhNjM3NzRlNDQ1MTMyNGU1NzU1MzI0ZTdhNTk3YTU5NTc0ZDMxNGY0NDQ1MzQ1YTU0NjczMTRlNDc1MTM0NTk1NzUyNmQ0ZTU0NDE3YTU5NmE2MzM1NGQ2YTZjNmI0ZjU0NTI2YzRlNmQ0OTc5NGU2YTQ5Nzc1YTY3M2QzZDIyMmMyMjYzNjg2MTY5NmU0OTQ0MjIzYTIyNGQ1MTNkM2QyMjJjMjI3NjY1NzI3MzY5NmY2ZTIyM2EzMTJjMjI3MzY5Njc2ZTYxNzQ3NTcyNjUyMjNhMjI1MjM5NDYyYjM0NTQ2MzUyNDE1YTM4NmQ3NzcxMzI0NTU5MzAzMTYzNTk2YzMzNzY2MjcxNmM0NjY1NzE3NjM4N2E3NjQ3NGE3NzVhNjgzMzU5NGQ0ZjU1NmI0MjM0NjQzNDUxNTc0ZTY2Mzc2NzQ0NjI2YzQ4NDgzMjU3NmI3MTYxNGE3NjYxNDg0NTc0NDM1NjYxNzA0OTcxMzM2NTM1NjU2MjM4NGU0MTc3M2QzZDIyN2Q=", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "hasScResults": true, + "receivers": [ + "000000000000000005008b60efce7fb25b140078645b71226cb9c644abc8a0d3" + ], + "receiversShardIDs": [ + 0 + ], + "operation": "transfer", + "function": "saveAttestation", + "isRelayed": true +} diff --git a/integrationtests/testdata/relayedTx/relayed-tx-intra.json b/integrationtests/testdata/relayedTx/relayed-tx-intra.json new file mode 100644 index 00000000..0e4ff0d0 --- /dev/null +++ b/integrationtests/testdata/relayedTx/relayed-tx-intra.json @@ -0,0 +1,28 @@ +{ + "miniBlockHash": "2709174224d13e49fd76a70b48bd3db7838ca715bcfe09be59cef043241d7ef3", + "nonce": 1196665, + "round": 50, + "value": "0", + "receiver": "6572643134657961796672766c72687a66727767357a776c65756132356d6b7a676e6367676e33356e766336786876357978776d6c326573306633646874", + "sender": "657264316b376a3665776a736c61347a73677638763666366665336476726b677633643064396a6572637a773435687a6564687965643873683275333475", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 15406000, + "gasUsed": 10556000, + "fee": "2257820000000000", + "data": "cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMwMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjcyNmI2ZTUzNGE0NzdhMzQzNzY5NTM0ZTc5NGI0MzY0MmY1MDRmNzE3MDc1Nzc2YjU0Nzc2ODQ1MzQzMDZkN2E0NzZhNTg1YTUxNjg2ZTYyMmI3MjRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzMzMjMzMzIzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI1NTMyNDYzMjVhNTU3NDZjNjU1NjVhNjg2MjQ4NTY2YzUxNDQ1OTc5NGU2YjU1MzI0ZDZiNDEzMjRkNmE1YTQ2NGU2YTQ5N2E0ZDU0NGQzNTRlNmE1NTMyNGQ1NDRkMzI0ZTZiNGQzMjUxNTQ2Mzc4NGQ3YTZiMzI1MTdhNjMzMDRlNmE1NTMzNGU0NDYzMzA0ZTdhNjczMjRlNTQ0ZDMzNGU3YTUxN2E0ZTdhNTkzMzRlNmI1NTdhNGQ0NDYzNzc0ZDdhNDk3YTRmNTQ2NDQyNGU3YTU5MzM0ZTU0NWE0NDRlNmE0NTMzNGU1NDU5MzI0ZTZhNjMzMzRlNTQ0ZDMwNGU3YTQ1N2E0ZTdhNTkzMzRlN2E0YTQxNGU2YTU1MzM0ZTQ0NTkzNDUxNDQ0ZDc3NGU3YTY3N2E0ZDU0NGQzMDRlNDQ1MTMyNGQ2YTRkNzg0ZTZhNTU3YTRkNDQ0ZDMxNGU2YTU5N2E0ZTU0NTE3YTRlNmE0NTdhNGU1NDRkMzE0ZTZhNDk3YTRkNTQ0ZDdhNGU0NDQ1MzA0ZDdhNTk3ODRkN2E2MzMyNGQ3YTRkMzA0ZTZhNGQ3YTRkNDQ0ZDc4NGU2YTU5N2E0ZTU0NTkzMDRlNDQ1NTdhNGU1NDRkMzE0ZTQ0NDk3YTRmNTQ1MTdhNGU0NDQ1MzI0ZTZhNTk3ODRkN2E2MzdhNGY1NTQxMzI0ZDZhNjMzMDRlNmE0ZTQxNGU2YTQ5MzI0ZDdhNGQ3ODRlN2E0NTMyNGQ1NDU5MzE0ZDdhNTU3YTRmNDQ0ZDdhNGQ3YTRkMzM0ZDZhNjM3OTRlN2E1OTdhNGU0NDRkNzc0ZDdhNDE3YTRlNTQ1YTQ0NGU2YTU5MzI1MjQ0NGQzMTRlN2E1OTMyNTI0NDRkMzI0ZTdhNTUzMjUxNTQ2MzMyNGU2YTUxN2E0ZTQ0NTk3YTRlN2E1MTdhNGY1NDVhNDI0ZTZhNGQ3YTRlNTQ1YTQyNGU3YTYzMzM1MTU0NGQzNDRlNmE1NTMzNGY0NDYzNzcyMjJjMjI2MzY4NjE2OTZlNDk0NDIyM2EyMjRkNTEzZDNkMjIyYzIyNzY2NTcyNzM2OTZmNmUyMjNhMzEyYzIyNzM2OTY3NmU2MTc0NzU3MjY1MjIzYTIyNzE2NjcwNGE0Nzc2NzM0NDQ0NDI1NTUxNGUyZjUyNTU0NzRmNTA1Mzc1NTIzMjQ4NGY0YTYxNGI3MDM4NDUzNjYzNGU1NDc3MzAzMzQzMzc2OTM0NTU3Nzc2MmY0YzU0NzM2ZDJiNmE3MDQyMzk3NTZjNDgzOTY2NTMyYjQ0NzE2MTcyNzE0ZjYyNDg0MTcwMzg2NjZkNzIzMDZhNDE1NTMxNzM2ZTM1NDE2NzNkM2QyMjdk", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "hasScResults": true, + "receivers": [ + "ae49d2246cf8ee248dc8a09dfcf3aaa6ec244f0844e349b31a35d94219dbfab3" + ], + "receiversShardIDs": [ + 0 + ], + "operation": "SaveKeyValue", + "isRelayed": true +} diff --git a/integrationtests/testdata/relayedTx/relayed-tx-source.json b/integrationtests/testdata/relayedTx/relayed-tx-source.json new file mode 100644 index 00000000..795ea663 --- /dev/null +++ b/integrationtests/testdata/relayedTx/relayed-tx-source.json @@ -0,0 +1,29 @@ +{ + "miniBlockHash": "fed7c174a849c30b88c36a26453407f1b95970941d0872e603e641c5c804104a", + "nonce": 1196667, + "round": 50, + "value": "0", + "receiver": "6572643134657961796672766c72687a66727767357a776c65756132356d6b7a676e6367676e33356e766336786876357978776d6c326573306633646874", + "sender": "657264316b376a3665776a736c61347a73677638763666366665336476726b677633643064396a6572637a773435687a6564687965643873683275333475", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 16610000, + "gasUsed": 16610000, + "fee": "1760000000000000", + "data": "cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMyMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjQxNDE0MTQxNDE0MTQxNDE0MTQxNDE0NjQxNDk3NDY3MzczODM1MmY3MzZjNzM1NTQxNDg2ODZiNTczMzQ1Njk2MjRjNmU0NzUyNGI3NjQ5NmY0ZTRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzUzMDMwMzAzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI2MzMyNDYzMjVhNTU0NjMwNjQ0NzU2N2E2NDQ3NDYzMDYxNTczOTc1NTE0NDQ2Njg1OTdhNDkzMTRkNmE1OTM1NTk2ZDUxMzM1YTQ0NDk3NzU5MzI0YTY5NTk1NDRkMzE1OTZkNTY2YzRmNDQ1OTMxNGQ0NDY0Njg0ZjU3NGU2YTRlN2E2NzdhNWE0NzU1Nzc0ZjQ0NWE2OTRlNDQ0NTMzNGU1NDZiMzQ1YTU0NTE3YTU5NTQ0ZTZiNWE2YTU2NmE1OTMyNDU3OTVhNTQ2ODY4NGQ2YTZjNDE0ZDZhNTEzNDRlNTQ2NzdhNGQ1NzRlNmQ0ZDU0NDUzMDRkNTQ1NjZkNTk2YTQxMzU0ZDZhNjM3NzRlNDQ1MTMyNGU1NzU1MzI0ZTdhNTk3YTU5NTc0ZDMxNGY0NDQ1MzQ1YTU0NjczMTRlNDc1MTM0NTk1NzUyNmQ0ZTU0NDE3YTU5NmE2MzM1NGQ2YTZjNmI0ZjU0NTI2YzRlNmQ0OTc5NGU2YTQ5Nzc1YTY3M2QzZDIyMmMyMjYzNjg2MTY5NmU0OTQ0MjIzYTIyNGQ1MTNkM2QyMjJjMjI3NjY1NzI3MzY5NmY2ZTIyM2EzMTJjMjI3MzY5Njc2ZTYxNzQ3NTcyNjUyMjNhMjI1MjM5NDYyYjM0NTQ2MzUyNDE1YTM4NmQ3NzcxMzI0NTU5MzAzMTYzNTk2YzMzNzY2MjcxNmM0NjY1NzE3NjM4N2E3NjQ3NGE3NzVhNjgzMzU5NGQ0ZjU1NmI0MjM0NjQzNDUxNTc0ZTY2Mzc2NzQ0NjI2YzQ4NDgzMjU3NmI3MTYxNGE3NjYxNDg0NTc0NDM1NjYxNzA0OTcxMzM2NTM1NjU2MjM4NGU0MTc3M2QzZDIyN2Q=", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "hasScResults": true, + "receivers": [ + "000000000000000005008b60efce7fb25b140078645b71226cb9c644abc8a0d3" + ], + "receiversShardIDs": [ + 0 + ], + "operation": "transfer", + "function": "saveAttestation", + "isRelayed": true +} diff --git a/integrationtests/testdata/scCallIntraShard/claim-rewards.json b/integrationtests/testdata/scCallIntraShard/claim-rewards.json new file mode 100644 index 00000000..abc9c6dc --- /dev/null +++ b/integrationtests/testdata/scCallIntraShard/claim-rewards.json @@ -0,0 +1,21 @@ +{ + "miniBlockHash": "60b38b11110d28d1b361359f9688bb041bb9180219a612a83ff00dcc0db4d607", + "nonce": 101, + "round": 50, + "value": "0", + "receiver": "65726431717171717171717171717171717067717877616b7432673775396174736e723033677163676d68637633387074376d6b64393471367368757774", + "sender": "65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 250000000, + "gasUsed": 33891715, + "fee": "406237150000000", + "data": "Y2xhaW1SZXdhcmRz", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "hasScResults": true, + "operation": "transfer" +} diff --git a/integrationtests/testdata/scCallIntraShard/sc-call-fail.json b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json new file mode 100644 index 00000000..da9b0e63 --- /dev/null +++ b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json @@ -0,0 +1,21 @@ +{ + "miniBlockHash": "5d04f80b044352bfbbde123702323eae07fdd8ca77f24f256079006058b6e7b4", + "nonce": 46, + "round": 50, + "value": "5000000000000000000", + "receiver": "6572643171717171717171717171717171717170717171717171717171717171717171717171717171717171717171717166686c6c6c6c73637274353672", + "sender": "65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 12000000, + "gasUsed": 12000000, + "fee": "181380000000000", + "data": "ZGVsZWdhdGU=", + "signature": "", + "timestamp": 5040, + "status": "fail", + "searchOrder": 0, + "hasScResults": true, + "operation": "transfer" +} diff --git a/integrationtests/testdata/transactions/move-balance.json b/integrationtests/testdata/transactions/move-balance.json new file mode 100644 index 00000000..d869b05f --- /dev/null +++ b/integrationtests/testdata/transactions/move-balance.json @@ -0,0 +1,20 @@ +{ + "miniBlockHash": "24c374c9405540e88a36959ea83eede6ad50f6872f82d2e2a2280975615e1811", + "nonce": 1, + "round": 50, + "value": "1234", + "receiver": "7265636569766572", + "sender": "73656e646572", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 70000, + "gasUsed": 62000, + "fee": "62000000000000", + "data": "dHJhbnNmZXI=", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "operation": "transfer" +} diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 2097da33..dbaa0cf6 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -16,8 +16,6 @@ import ( "github.com/stretchr/testify/require" ) -const moveBalanceTransaction = `{"miniBlockHash":"24c374c9405540e88a36959ea83eede6ad50f6872f82d2e2a2280975615e1811","nonce":1,"round":50,"value":"1234","receiver":"7265636569766572","sender":"73656e646572","receiverShard":0,"senderShard":0,"gasPrice":1000000000,"gasLimit":70000,"gasUsed":62000,"fee":"62000000000000","data":"dHJhbnNmZXI=","signature":"","timestamp":5040,"status":"success","searchOrder":0,"operation":"transfer"}` - func TestElasticIndexerSaveTransactions(t *testing.T) { setLogLevelDebug() @@ -54,7 +52,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { GasPrice: 1000000000, Data: []byte("transfer"), Value: big.NewInt(1234), - }, 0, big.NewInt(0)), + }, 62000, big.NewInt(62000000000000)), }, } err = esProc.SaveTransactions(body, header, pool, nil) @@ -65,5 +63,8 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) - compareTxs(t, []byte(moveBalanceTransaction), genericResponse.Docs[0].Source) + require.JSONEq(t, + readExpectedResult("./testdata/transactions/move-balance.json"), + string(genericResponse.Docs[0].Source), + ) } From 846d81e55dbb2923a87ec1ac1af91cdcdd652a6e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 29 Aug 2022 14:49:29 +0300 Subject: [PATCH 27/96] fix integration tests --- go.mod | 2 +- go.sum | 4 +- integrationtests/accountsESDTRollback_test.go | 39 ++++++------ integrationtests/claimRewards_test.go | 5 +- integrationtests/esdtTransfer_test.go | 6 +- integrationtests/logsCrossShard_test.go | 18 +++--- .../nftTransferCrossShard_test.go | 63 +++++++++++-------- .../nftTransferCrossWithScCall_test.go | 21 ++++--- integrationtests/relayedTx_test.go | 9 ++- integrationtests/scCallIntraShard_test.go | 42 +++++++------ .../testdata/esdtTransfer/esdt-transfer.json | 1 + .../tx-complete-with-status.json | 2 +- .../cross-shard-transfer-with-sc-call.json | 1 + .../relayedTx/relayed-tx-after-refund.json | 1 + .../testdata/relayedTx/relayed-tx-intra.json | 1 + .../testdata/relayedTx/relayed-tx-source.json | 1 + .../scCallIntraShard/claim-rewards.json | 1 + .../scCallIntraShard/sc-call-fail.json | 1 + .../testdata/transactions/move-balance.json | 1 + integrationtests/transactions_test.go | 20 +++--- process/transactions/transactionDBBuilder.go | 1 + process/transactions/transactionsGrouper.go | 4 +- 22 files changed, 142 insertions(+), 102 deletions(-) diff --git a/go.mod b/go.mod index de95e3c1..3fcd2765 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220808113242-256bcac1fcfe + github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00 github.com/ElrondNetwork/elrond-go-logger v1.0.7 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index dc0f52c8..30b474dd 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220808113242-256bcac1fcfe h1:zKZzyT+D4CrTgJkkvXKROi/xaDN0WhalEt2B5BOZPGc= -github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220808113242-256bcac1fcfe/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00 h1:pH+/JIebRGRq5PaC+qZZ7ZBr+YWDDI/N+hZtCD4onUo= +github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7 h1:Ldl1rVS0RGKc1IsW8jIaGCb6Zwei04gsMvyjL05X6mE= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 2fd4afdd..46cbc747 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "math/big" "testing" @@ -13,9 +14,8 @@ import ( coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/esdt" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) @@ -25,8 +25,6 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - feeComputer := &mock.EconomicsHandlerMock{} - shardCoordinator := &mock.ShardCoordinatorMock{ SelfID: 1, ComputeIdCalled: func(address []byte) uint32 { @@ -41,27 +39,30 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { Creator: []byte("creator"), }, } - addr := "aaaabbbb" - mockAccount := &mock.UserAccountStub{ - RetrieveValueFromDataTrieTrackerCalled: func(key []byte) ([]byte, error) { - return json.Marshal(esdtToken) - }, - AddressBytesCalled: func() []byte { - return []byte(addr) - }, - } - accounts := &mock.AccountsStub{ - LoadAccountCalled: func(container []byte) (vmcommon.AccountHandler, error) { - return mockAccount, nil + addr := hex.EncodeToString([]byte("aaaabbbb")) + coreAlteredAccounts := map[string]*outport.AlteredAccount{ + addr: { + Address: addr, + Tokens: []*outport.AccountTokenData{ + { + Identifier: "TOKEN-eeee", + Nonce: 2, + Balance: "1000", + MetaData: &esdt.MetaData{ + Creator: []byte("creator"), + }, + Properties: "ok", + }, + }, }, } - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) // CREATE SEMI-FUNGIBLE TOKEN esdtDataBytes, _ := json.Marshal(esdtToken) - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { TxHash: "h1", @@ -85,7 +86,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { TimeStamp: 5040, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) require.Nil(t, err) ids := []string{"6161616162626262-TOKEN-eeee-02"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index ac40d695..6349b963 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -89,9 +89,12 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { Value: big.NewInt(0), } + tx := outport.NewTransactionHandlerWithGasAndFee(tx1, 1068000, big.NewInt(78000000000000)) + tx.SetInitialPaidFee(big.NewInt(127320000000000)) + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(tx1, 1068000, big.NewInt(78000000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 4c115ceb..19f248fa 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -83,9 +83,13 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { OriginalTxHash: txHash, } + initialPaidFee, _ := big.NewInt(0).SetString("104000110000000", 10) + tx := outport.NewTransactionHandlerWithGasAndFee(txESDT, 104011, big.NewInt(104000110000000)) + tx.SetInitialPaidFee(initialPaidFee) + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(txESDT, 104011, big.NewInt(104000110000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 93018f23..fb2e6a4c 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -12,7 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go-core/data/indexer" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -23,11 +23,9 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - accounts := &mock.AccountsStub{} - feeComputer := &mock.EconomicsHandlerMock{} shardCoordinator := &mock.ShardCoordinatorMock{} - esProc, err := CreateElasticProcessor(esClient, accounts, shardCoordinator, feeComputer) + esProc, err := CreateElasticProcessor(esClient, shardCoordinator) require.Nil(t, err) header := &dataBlock.Header{ @@ -37,7 +35,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { body := &dataBlock.Body{} // INDEX ON SOURCE - pool := &indexer.Pool{ + pool := &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ @@ -55,7 +53,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) ids := []string{hex.EncodeToString([]byte("cross-log"))} @@ -72,7 +70,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { Round: 50, TimeStamp: 6040, } - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ @@ -96,7 +94,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) @@ -111,7 +109,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { Round: 50, TimeStamp: 5000, } - pool = &indexer.Pool{ + pool = &outport.Pool{ Logs: []*coreData.LogData{ { LogHandler: &transaction.Log{ @@ -129,7 +127,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index c38400a2..51508854 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -63,17 +63,20 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { OriginalTxHash: txHash, } + tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 79, + SndAddr: []byte("erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50"), + RcvAddr: []byte("erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50"), + GasLimit: 150000000, + GasPrice: 1000000000, + Data: []byte("ESDTNFTTransfer@4c4b4641524d2d336634663962@016534@6f1e6f01bc7627f5ae@00000000000000000500f1c8f2fdc58a63c6b201fc2ed629962d3dfa33fe7ceb@636f6d706f756e645265776172647350726f7879@000000000000000005004f79ec44bb13372b5ac9d996d749120f476427627ceb"), + Value: big.NewInt(0), + }, 150000000, big.NewInt(1904415000000000)) + tx.SetInitialPaidFee(big.NewInt(1904415000000000)) + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 79, - SndAddr: []byte("erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50"), - RcvAddr: []byte("erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50"), - GasLimit: 150000000, - GasPrice: 1000000000, - Data: []byte("ESDTNFTTransfer@4c4b4641524d2d336634663962@016534@6f1e6f01bc7627f5ae@00000000000000000500f1c8f2fdc58a63c6b201fc2ed629962d3dfa33fe7ceb@636f6d706f756e645265776172647350726f7879@000000000000000005004f79ec44bb13372b5ac9d996d749120f476427627ceb"), - Value: big.NewInt(0), - }, 150000000, big.NewInt(1904415000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), @@ -185,18 +188,21 @@ func TestNFTTransferCrossShard(t *testing.T) { OriginalTxHash: txHash, } + tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 79, + SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + GasLimit: 5000000, + GasPrice: 1000000000, + Data: []byte("ESDTNFTTransfer@536f6d657468696e672d616263646566@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), + Value: big.NewInt(0), + }, 963500, big.NewInt(235850000000000)) + tx.SetInitialPaidFee(big.NewInt(276215000000000)) + refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 79, - SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - GasLimit: 5000000, - GasPrice: 1000000000, - Data: []byte("ESDTNFTTransfer@536f6d657468696e672d616263646566@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), - Value: big.NewInt(0), - }, 963500, big.NewInt(235850000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ @@ -348,18 +354,21 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t }, } + tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 79, + SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + GasLimit: 5000000, + GasPrice: 1000000000, + Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), + Value: big.NewInt(0), + }, 963500, big.NewInt(238820000000000)) + tx.SetInitialPaidFee(big.NewInt(595490000000000)) + refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 79, - SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - GasLimit: 5000000, - GasPrice: 1000000000, - Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), - Value: big.NewInt(0), - }, 963500, big.NewInt(238820000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 0b87145f..70b4448d 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -61,18 +61,21 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { OriginalTxHash: txHash, } + tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 79, + SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + GasLimit: 5000000, + GasPrice: 1000000000, + Data: []byte("ESDTNFTTransfer@4d45584641524d2d636362323532@078b@0347543e5b59c9be8670@0801120b000347543e5b59c9be86702266088b0f1a20000000000000000005005754e4f6ba0b94efd71a0e4dd4814ee24e5f75297ceb32003a3d0000000701b6408636587c0000000000000410000000000000041001000000000a0347543e5b59c9be8670000000000000000a0347543e5b59c9be8670@636c61696d52657761726473"), + Value: big.NewInt(0), + }, 5000000, big.NewInt(595490000000000)) + tx.SetInitialPaidFee(big.NewInt(595490000000000)) + // refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 79, - SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - RcvAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - GasLimit: 5000000, - GasPrice: 1000000000, - Data: []byte("ESDTNFTTransfer@4d45584641524d2d636362323532@078b@0347543e5b59c9be8670@0801120b000347543e5b59c9be86702266088b0f1a20000000000000000005005754e4f6ba0b94efd71a0e4dd4814ee24e5f75297ceb32003a3d0000000701b6408636587c0000000000000410000000000000041001000000000a0347543e5b59c9be8670000000000000000a0347543e5b59c9be8670@636c61696d52657761726473"), - Value: big.NewInt(0), - }, 5000000, big.NewInt(595490000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index ac037c34..1e746190 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -73,9 +73,12 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { OriginalTxHash: txHash, } + tx := outport.NewTransactionHandlerWithGasAndFee(initialTx, 16610000, big.NewInt(1760000000000000)) + tx.SetInitialPaidFee(big.NewInt(1760000000000000)) + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 16610000, big.NewInt(1760000000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), @@ -203,9 +206,11 @@ func TestRelayedTransactionIntraShard(t *testing.T) { Value: refundValueBig, } + tx := outport.NewTransactionHandlerWithGasAndFee(initialTx, 10556000, big.NewInt(2257820000000000)) + tx.SetInitialPaidFee(big.NewInt(2306320000000000)) pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(initialTx, 10556000, big.NewInt(2257820000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 0ee770a9..ad023aba 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -46,17 +46,20 @@ func TestTransactionWithSCCallFail(t *testing.T) { } refundValueBig, _ := big.NewInt(0).SetString("5000000000000000000", 10) + tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 46, + SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + RcvAddr: []byte("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqfhllllscrt56r"), + GasLimit: 12000000, + GasPrice: 1000000000, + Data: []byte("delegate"), + Value: refundValueBig, + }, 12000000, big.NewInt(181380000000000)) + tx.SetInitialPaidFee(big.NewInt(181380000000000)) + pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 46, - SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - RcvAddr: []byte("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqfhllllscrt56r"), - GasLimit: 12000000, - GasPrice: 1000000000, - Data: []byte("delegate"), - Value: refundValueBig, - }, 12000000, big.NewInt(181380000000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ @@ -114,18 +117,21 @@ func TestTransactionWithScCallSuccess(t *testing.T) { }, } + tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 101, + SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), + RcvAddr: []byte("erd1qqqqqqqqqqqqqpgqxwakt2g7u9atsnr03gqcgmhcv38pt7mkd94q6shuwt"), + GasLimit: 250000000, + GasPrice: 1000000000, + Data: []byte("claimRewards"), + Value: big.NewInt(0), + }, 33891715, big.NewInt(406237150000000)) + tx.SetInitialPaidFee(big.NewInt(2567320000000000)) + refundValueBig, _ := big.NewInt(0).SetString("2161082850000000", 10) pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 101, - SndAddr: []byte("erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv"), - RcvAddr: []byte("erd1qqqqqqqqqqqqqpgqxwakt2g7u9atsnr03gqcgmhcv38pt7mkd94q6shuwt"), - GasLimit: 250000000, - GasPrice: 1000000000, - Data: []byte("claimRewards"), - Value: big.NewInt(0), - }, 33891715, big.NewInt(406237150000000)), + string(txHash): tx, }, Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ diff --git a/integrationtests/testdata/esdtTransfer/esdt-transfer.json b/integrationtests/testdata/esdtTransfer/esdt-transfer.json index 1115aea8..187457e8 100644 --- a/integrationtests/testdata/esdtTransfer/esdt-transfer.json +++ b/integrationtests/testdata/esdtTransfer/esdt-transfer.json @@ -11,6 +11,7 @@ "gasLimit": 104011, "gasUsed": 104011, "fee": "104000110000000", + "initialPaidFee": "104000110000000", "data": "RVNEVFRyYW5zZmVyQDU0NDc0ZTJkMzgzODYyMzgzMzY2QDBh", "signature": "", "timestamp": 5040, diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json index 3632ab87..e06f49d3 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json @@ -29,6 +29,6 @@ "gasPrice": 1000000000, "timestamp": 5040, "status": "fail", - "initialPaidFee": "279185000000000", + "initialPaidFee": "595490000000000", "searchOrder": 0 } diff --git a/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json b/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json index cf498041..a4f806f4 100644 --- a/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json +++ b/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json @@ -11,6 +11,7 @@ "gasLimit": 5000000, "gasUsed": 5000000, "fee": "595490000000000", + "initialPaidFee": "595490000000000", "data": "RVNEVE5GVFRyYW5zZmVyQDRkNDU1ODQ2NDE1MjRkMmQ2MzYzNjIzMjM1MzJAMDc4YkAwMzQ3NTQzZTViNTljOWJlODY3MEAwODAxMTIwYjAwMDM0NzU0M2U1YjU5YzliZTg2NzAyMjY2MDg4YjBmMWEyMDAwMDAwMDAwMDAwMDAwMDAwNTAwNTc1NGU0ZjZiYTBiOTRlZmQ3MWEwZTRkZDQ4MTRlZTI0ZTVmNzUyOTdjZWIzMjAwM2EzZDAwMDAwMDA3MDFiNjQwODYzNjU4N2MwMDAwMDAwMDAwMDAwNDEwMDAwMDAwMDAwMDAwMDQxMDAxMDAwMDAwMDAwYTAzNDc1NDNlNWI1OWM5YmU4NjcwMDAwMDAwMDAwMDAwMDAwYTAzNDc1NDNlNWI1OWM5YmU4NjcwQDYzNmM2MTY5NmQ1MjY1Nzc2MTcyNjQ3Mw==", "signature": "", "timestamp": 5040, diff --git a/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json index 3df42223..55f9dc69 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json @@ -11,6 +11,7 @@ "gasLimit": 16610000, "gasUsed": 7982817, "fee": "1673728170000000", + "initialPaidFee": "1760000000000000", "data": "cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMyMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjQxNDE0MTQxNDE0MTQxNDE0MTQxNDE0NjQxNDk3NDY3MzczODM1MmY3MzZjNzM1NTQxNDg2ODZiNTczMzQ1Njk2MjRjNmU0NzUyNGI3NjQ5NmY0ZTRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzUzMDMwMzAzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI2MzMyNDYzMjVhNTU0NjMwNjQ0NzU2N2E2NDQ3NDYzMDYxNTczOTc1NTE0NDQ2Njg1OTdhNDkzMTRkNmE1OTM1NTk2ZDUxMzM1YTQ0NDk3NzU5MzI0YTY5NTk1NDRkMzE1OTZkNTY2YzRmNDQ1OTMxNGQ0NDY0Njg0ZjU3NGU2YTRlN2E2NzdhNWE0NzU1Nzc0ZjQ0NWE2OTRlNDQ0NTMzNGU1NDZiMzQ1YTU0NTE3YTU5NTQ0ZTZiNWE2YTU2NmE1OTMyNDU3OTVhNTQ2ODY4NGQ2YTZjNDE0ZDZhNTEzNDRlNTQ2NzdhNGQ1NzRlNmQ0ZDU0NDUzMDRkNTQ1NjZkNTk2YTQxMzU0ZDZhNjM3NzRlNDQ1MTMyNGU1NzU1MzI0ZTdhNTk3YTU5NTc0ZDMxNGY0NDQ1MzQ1YTU0NjczMTRlNDc1MTM0NTk1NzUyNmQ0ZTU0NDE3YTU5NmE2MzM1NGQ2YTZjNmI0ZjU0NTI2YzRlNmQ0OTc5NGU2YTQ5Nzc1YTY3M2QzZDIyMmMyMjYzNjg2MTY5NmU0OTQ0MjIzYTIyNGQ1MTNkM2QyMjJjMjI3NjY1NzI3MzY5NmY2ZTIyM2EzMTJjMjI3MzY5Njc2ZTYxNzQ3NTcyNjUyMjNhMjI1MjM5NDYyYjM0NTQ2MzUyNDE1YTM4NmQ3NzcxMzI0NTU5MzAzMTYzNTk2YzMzNzY2MjcxNmM0NjY1NzE3NjM4N2E3NjQ3NGE3NzVhNjgzMzU5NGQ0ZjU1NmI0MjM0NjQzNDUxNTc0ZTY2Mzc2NzQ0NjI2YzQ4NDgzMjU3NmI3MTYxNGE3NjYxNDg0NTc0NDM1NjYxNzA0OTcxMzM2NTM1NjU2MjM4NGU0MTc3M2QzZDIyN2Q=", "signature": "", "timestamp": 5040, diff --git a/integrationtests/testdata/relayedTx/relayed-tx-intra.json b/integrationtests/testdata/relayedTx/relayed-tx-intra.json index 0e4ff0d0..85f3219a 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-intra.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-intra.json @@ -11,6 +11,7 @@ "gasLimit": 15406000, "gasUsed": 10556000, "fee": "2257820000000000", + "initialPaidFee":"2306320000000000", "data": "cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMwMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjcyNmI2ZTUzNGE0NzdhMzQzNzY5NTM0ZTc5NGI0MzY0MmY1MDRmNzE3MDc1Nzc2YjU0Nzc2ODQ1MzQzMDZkN2E0NzZhNTg1YTUxNjg2ZTYyMmI3MjRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzMzMjMzMzIzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI1NTMyNDYzMjVhNTU3NDZjNjU1NjVhNjg2MjQ4NTY2YzUxNDQ1OTc5NGU2YjU1MzI0ZDZiNDEzMjRkNmE1YTQ2NGU2YTQ5N2E0ZDU0NGQzNTRlNmE1NTMyNGQ1NDRkMzI0ZTZiNGQzMjUxNTQ2Mzc4NGQ3YTZiMzI1MTdhNjMzMDRlNmE1NTMzNGU0NDYzMzA0ZTdhNjczMjRlNTQ0ZDMzNGU3YTUxN2E0ZTdhNTkzMzRlNmI1NTdhNGQ0NDYzNzc0ZDdhNDk3YTRmNTQ2NDQyNGU3YTU5MzM0ZTU0NWE0NDRlNmE0NTMzNGU1NDU5MzI0ZTZhNjMzMzRlNTQ0ZDMwNGU3YTQ1N2E0ZTdhNTkzMzRlN2E0YTQxNGU2YTU1MzM0ZTQ0NTkzNDUxNDQ0ZDc3NGU3YTY3N2E0ZDU0NGQzMDRlNDQ1MTMyNGQ2YTRkNzg0ZTZhNTU3YTRkNDQ0ZDMxNGU2YTU5N2E0ZTU0NTE3YTRlNmE0NTdhNGU1NDRkMzE0ZTZhNDk3YTRkNTQ0ZDdhNGU0NDQ1MzA0ZDdhNTk3ODRkN2E2MzMyNGQ3YTRkMzA0ZTZhNGQ3YTRkNDQ0ZDc4NGU2YTU5N2E0ZTU0NTkzMDRlNDQ1NTdhNGU1NDRkMzE0ZTQ0NDk3YTRmNTQ1MTdhNGU0NDQ1MzI0ZTZhNTk3ODRkN2E2MzdhNGY1NTQxMzI0ZDZhNjMzMDRlNmE0ZTQxNGU2YTQ5MzI0ZDdhNGQ3ODRlN2E0NTMyNGQ1NDU5MzE0ZDdhNTU3YTRmNDQ0ZDdhNGQ3YTRkMzM0ZDZhNjM3OTRlN2E1OTdhNGU0NDRkNzc0ZDdhNDE3YTRlNTQ1YTQ0NGU2YTU5MzI1MjQ0NGQzMTRlN2E1OTMyNTI0NDRkMzI0ZTdhNTUzMjUxNTQ2MzMyNGU2YTUxN2E0ZTQ0NTk3YTRlN2E1MTdhNGY1NDVhNDI0ZTZhNGQ3YTRlNTQ1YTQyNGU3YTYzMzM1MTU0NGQzNDRlNmE1NTMzNGY0NDYzNzcyMjJjMjI2MzY4NjE2OTZlNDk0NDIyM2EyMjRkNTEzZDNkMjIyYzIyNzY2NTcyNzM2OTZmNmUyMjNhMzEyYzIyNzM2OTY3NmU2MTc0NzU3MjY1MjIzYTIyNzE2NjcwNGE0Nzc2NzM0NDQ0NDI1NTUxNGUyZjUyNTU0NzRmNTA1Mzc1NTIzMjQ4NGY0YTYxNGI3MDM4NDUzNjYzNGU1NDc3MzAzMzQzMzc2OTM0NTU3Nzc2MmY0YzU0NzM2ZDJiNmE3MDQyMzk3NTZjNDgzOTY2NTMyYjQ0NzE2MTcyNzE0ZjYyNDg0MTcwMzg2NjZkNzIzMDZhNDE1NTMxNzM2ZTM1NDE2NzNkM2QyMjdk", "signature": "", "timestamp": 5040, diff --git a/integrationtests/testdata/relayedTx/relayed-tx-source.json b/integrationtests/testdata/relayedTx/relayed-tx-source.json index 795ea663..0a367ae7 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-source.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-source.json @@ -11,6 +11,7 @@ "gasLimit": 16610000, "gasUsed": 16610000, "fee": "1760000000000000", + "initialPaidFee": "1760000000000000", "data": "cmVsYXllZFR4QDdiMjI2ZTZmNmU2MzY1MjIzYTMyMmMyMjc2NjE2Yzc1NjUyMjNhMzAyYzIyNzI2NTYzNjU2OTc2NjU3MjIyM2EyMjQxNDE0MTQxNDE0MTQxNDE0MTQxNDE0NjQxNDk3NDY3MzczODM1MmY3MzZjNzM1NTQxNDg2ODZiNTczMzQ1Njk2MjRjNmU0NzUyNGI3NjQ5NmY0ZTRkM2QyMjJjMjI3MzY1NmU2NDY1NzIyMjNhMjI3MjZiNmU1MzRhNDc3YTM0Mzc2OTUzNGU3OTRiNDM2NDJmNTA0ZjcxNzA3NTc3NmI1NDc3Njg0NTM0MzA2ZDdhNDc2YTU4NWE1MTY4NmU2MjJiNzI0ZDNkMjIyYzIyNjc2MTczNTA3MjY5NjM2NTIyM2EzMTMwMzAzMDMwMzAzMDMwMzAzMDJjMjI2NzYxNzM0YzY5NmQ2OTc0MjIzYTMxMzUzMDMwMzAzMDMwMzAyYzIyNjQ2MTc0NjEyMjNhMjI2MzMyNDYzMjVhNTU0NjMwNjQ0NzU2N2E2NDQ3NDYzMDYxNTczOTc1NTE0NDQ2Njg1OTdhNDkzMTRkNmE1OTM1NTk2ZDUxMzM1YTQ0NDk3NzU5MzI0YTY5NTk1NDRkMzE1OTZkNTY2YzRmNDQ1OTMxNGQ0NDY0Njg0ZjU3NGU2YTRlN2E2NzdhNWE0NzU1Nzc0ZjQ0NWE2OTRlNDQ0NTMzNGU1NDZiMzQ1YTU0NTE3YTU5NTQ0ZTZiNWE2YTU2NmE1OTMyNDU3OTVhNTQ2ODY4NGQ2YTZjNDE0ZDZhNTEzNDRlNTQ2NzdhNGQ1NzRlNmQ0ZDU0NDUzMDRkNTQ1NjZkNTk2YTQxMzU0ZDZhNjM3NzRlNDQ1MTMyNGU1NzU1MzI0ZTdhNTk3YTU5NTc0ZDMxNGY0NDQ1MzQ1YTU0NjczMTRlNDc1MTM0NTk1NzUyNmQ0ZTU0NDE3YTU5NmE2MzM1NGQ2YTZjNmI0ZjU0NTI2YzRlNmQ0OTc5NGU2YTQ5Nzc1YTY3M2QzZDIyMmMyMjYzNjg2MTY5NmU0OTQ0MjIzYTIyNGQ1MTNkM2QyMjJjMjI3NjY1NzI3MzY5NmY2ZTIyM2EzMTJjMjI3MzY5Njc2ZTYxNzQ3NTcyNjUyMjNhMjI1MjM5NDYyYjM0NTQ2MzUyNDE1YTM4NmQ3NzcxMzI0NTU5MzAzMTYzNTk2YzMzNzY2MjcxNmM0NjY1NzE3NjM4N2E3NjQ3NGE3NzVhNjgzMzU5NGQ0ZjU1NmI0MjM0NjQzNDUxNTc0ZTY2Mzc2NzQ0NjI2YzQ4NDgzMjU3NmI3MTYxNGE3NjYxNDg0NTc0NDM1NjYxNzA0OTcxMzM2NTM1NjU2MjM4NGU0MTc3M2QzZDIyN2Q=", "signature": "", "timestamp": 5040, diff --git a/integrationtests/testdata/scCallIntraShard/claim-rewards.json b/integrationtests/testdata/scCallIntraShard/claim-rewards.json index abc9c6dc..0a1731b4 100644 --- a/integrationtests/testdata/scCallIntraShard/claim-rewards.json +++ b/integrationtests/testdata/scCallIntraShard/claim-rewards.json @@ -10,6 +10,7 @@ "gasPrice": 1000000000, "gasLimit": 250000000, "gasUsed": 33891715, + "initialPaidFee": "2567320000000000", "fee": "406237150000000", "data": "Y2xhaW1SZXdhcmRz", "signature": "", diff --git a/integrationtests/testdata/scCallIntraShard/sc-call-fail.json b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json index da9b0e63..12707b18 100644 --- a/integrationtests/testdata/scCallIntraShard/sc-call-fail.json +++ b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json @@ -11,6 +11,7 @@ "gasLimit": 12000000, "gasUsed": 12000000, "fee": "181380000000000", + "initialPaidFee": "181380000000000", "data": "ZGVsZWdhdGU=", "signature": "", "timestamp": 5040, diff --git a/integrationtests/testdata/transactions/move-balance.json b/integrationtests/testdata/transactions/move-balance.json index d869b05f..abe7d40a 100644 --- a/integrationtests/testdata/transactions/move-balance.json +++ b/integrationtests/testdata/transactions/move-balance.json @@ -11,6 +11,7 @@ "gasLimit": 70000, "gasUsed": 62000, "fee": "62000000000000", + "initialPaidFee": "62080000000000", "data": "dHJhbnNmZXI=", "signature": "", "timestamp": 5040, diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index dbaa0cf6..83781598 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -42,17 +42,19 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { }, }, } + tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 1, + SndAddr: []byte("sender"), + RcvAddr: []byte("receiver"), + GasLimit: 70000, + GasPrice: 1000000000, + Data: []byte("transfer"), + Value: big.NewInt(1234), + }, 62000, big.NewInt(62000000000000)) + tx.SetInitialPaidFee(big.NewInt(62080000000000)) pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 1, - SndAddr: []byte("sender"), - RcvAddr: []byte("receiver"), - GasLimit: 70000, - GasPrice: 1000000000, - Data: []byte("transfer"), - Value: big.NewInt(1234), - }, 62000, big.NewInt(62000000000000)), + string(txHash): tx, }, } err = esProc.SaveTransactions(body, header, pool, nil) diff --git a/process/transactions/transactionDBBuilder.go b/process/transactions/transactionDBBuilder.go index 3da22c7c..a4d4f47a 100644 --- a/process/transactions/transactionDBBuilder.go +++ b/process/transactions/transactionDBBuilder.go @@ -46,6 +46,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( txStatus string, fee *big.Int, gasUsed uint64, + initialPaidFee *big.Int, ) *data.Transaction { isScCall := core.IsSmartContractAddress(tx.RcvAddr) res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr) diff --git a/process/transactions/transactionsGrouper.go b/process/transactions/transactionsGrouper.go index 25d45761..b7a2be0f 100644 --- a/process/transactions/transactionsGrouper.go +++ b/process/transactions/transactionsGrouper.go @@ -110,7 +110,7 @@ func (tg *txsGrouper) prepareNormalTxForDB( return nil, false } - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, mbStatus, txHandler.GetFee(), txHandler.GetGasUsed()) + dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, mbStatus, txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee()) return dbTx, true } @@ -212,7 +212,7 @@ func (tg *txsGrouper) prepareInvalidTxForDB( return nil, false } - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), txHandler.GetFee(), txHandler.GetGasUsed()) + dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee()) return dbTx, true } From 9fa91c5a995ca95c7b3e65566cd188d011921c2b Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 29 Aug 2022 14:52:11 +0300 Subject: [PATCH 28/96] fix unit tests --- process/transactions/transactionDBBuilder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/transactions/transactionDBBuilder_test.go b/process/transactions/transactionDBBuilder_test.go index 062f555e..5d471dad 100644 --- a/process/transactions/transactionDBBuilder_test.go +++ b/process/transactions/transactionDBBuilder_test.go @@ -78,7 +78,7 @@ func TestGetMoveBalanceTransaction(t *testing.T) { Receivers: []string{}, } - dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500) + dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500, big.NewInt(100)) require.Equal(t, expectedTx, dbTx) } From b1a2f2fdb242c4488671d881ff100fad2a3bca7e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 29 Aug 2022 17:39:42 +0300 Subject: [PATCH 29/96] fixes after review --- client/data.go | 2 +- client/elasticClient.go | 2 +- converters/code.go | 2 +- data/buffer.go | 2 +- data/transaction.go | 2 +- errors.go | 3 --- integrationtests/utils.go | 15 --------------- mock/dbTransactionsHandlerStub.go | 1 + process/elasticProcessor.go | 1 + process/logsevents/serialize.go | 4 ++-- process/miniblocks/serialize.go | 2 +- process/miniblocks/serialize_test.go | 2 +- process/tags/serialize.go | 3 ++- process/tags/serialize_test.go | 2 +- .../transactions/transactionsProcessor_test.go | 2 +- tools/clusters-checker/pkg/checkers/interface.go | 2 +- tools/index-modifier/pkg/client/client.go | 2 +- 17 files changed, 17 insertions(+), 32 deletions(-) diff --git a/client/data.go b/client/data.go index 9604f73f..def58106 100644 --- a/client/data.go +++ b/client/data.go @@ -18,7 +18,7 @@ type BulkRequestResponse struct { } `json:"items"` } -// Item defines the structure of a item from a bulk response +// Item defines the structure of an item from a bulk response type Item struct { Index string `json:"_index"` ID string `json:"_id"` diff --git a/client/elasticClient.go b/client/elasticClient.go index 76b2d60a..236e97f2 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -125,7 +125,7 @@ func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string) error { return elasticBulkRequestResponseHandler(res) } -// DoMultiGet wil do a multi get request to elaticsearch server +// DoMultiGet wil do a multi get request to Elasticsearch server func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, resBody interface{}) error { obj := getDocumentsByIDsQuery(ids, withSource) body, err := encode(obj) diff --git a/converters/code.go b/converters/code.go index 85178623..82382eb8 100644 --- a/converters/code.go +++ b/converters/code.go @@ -5,7 +5,7 @@ import "strings" // TODO use FormatPainlessSource everywhere // FormatPainlessSource will clean all new lines and white spaces from the input -// Input example: +// example: /*` if (!ctx._source.containsKey('roles')) { ctx._source.roles = new HashMap(); diff --git a/data/buffer.go b/data/buffer.go index 58469a33..f8d6ad73 100644 --- a/data/buffer.go +++ b/data/buffer.go @@ -2,7 +2,7 @@ package data import "bytes" -// DefaultMaxBulkSize is the constant for the maximum size of one bulk request that is sent to the elaticsearch database +// DefaultMaxBulkSize is the constant for the maximum size of one bulk request that is sent to the Elasticsearch database const DefaultMaxBulkSize = 4194304 // 4MB // BufferSlice extend structure bytes.Buffer with new methods diff --git a/data/transaction.go b/data/transaction.go index 260488ae..05b93a65 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -78,7 +78,7 @@ func (t *Transaction) GetValue() *big.Int { return bigIntValue } -// Receipt is a structure containing all the fields that need to be save for a Receipt +// Receipt is a structure containing all the fields that need to be safe for a Receipt type Receipt struct { Hash string `json:"-"` Value string `json:"value"` diff --git a/errors.go b/errors.go index c022e533..6c2e539b 100644 --- a/errors.go +++ b/errors.go @@ -35,9 +35,6 @@ var ErrNoElasticUrlProvided = errors.New("no elastic url provided") // ErrBackOff signals that an error was received from the server var ErrBackOff = errors.New("back off something is not working well") -// ErrNilTransactionFeeCalculator signals that a nil transaction fee calculator has been provided -var ErrNilTransactionFeeCalculator = errors.New("nil transaction fee calculator") - // ErrNilHasher signals that a nil hasher has been provided var ErrNilHasher = errors.New("nil hasher provided") diff --git a/integrationtests/utils.go b/integrationtests/utils.go index c04f3c6f..227d3c2e 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -4,18 +4,15 @@ import ( "encoding/json" "io/ioutil" "os" - "testing" indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/client" "github.com/ElrondNetwork/elastic-indexer-go/client/logging" - "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elastic-indexer-go/process" "github.com/ElrondNetwork/elastic-indexer-go/process/factory" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/elastic/go-elasticsearch/v7" - "github.com/stretchr/testify/require" ) const esURL = "http://localhost:9200" @@ -53,18 +50,6 @@ func CreateElasticProcessor( return factory.CreateElasticProcessor(args) } -func compareTxs(t *testing.T, expected []byte, actual []byte) { - expectedTx := &data.Transaction{} - err := json.Unmarshal(expected, expectedTx) - require.Nil(t, err) - - actualTx := &data.Transaction{} - err = json.Unmarshal(actual, actualTx) - require.Nil(t, err) - - require.Equal(t, expectedTx, actualTx) -} - func readExpectedResult(path string) string { jsonFile, _ := os.Open(path) byteValue, _ := ioutil.ReadAll(jsonFile) diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index de1162cd..1266286f 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -14,6 +14,7 @@ type DBTransactionProcessorStub struct { SerializeScResultsCalled func(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } +// SerializeTransactionsFeeData - func (tps *DBTransactionProcessorStub) SerializeTransactionsFeeData(_ map[string]*data.FeeData, _ *data.BufferSlice, _ string) error { return nil } diff --git a/process/elasticProcessor.go b/process/elasticProcessor.go index e5fe68a6..89731e3b 100644 --- a/process/elasticProcessor.go +++ b/process/elasticProcessor.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" diff --git a/process/logsevents/serialize.go b/process/logsevents/serialize.go index 68269585..e7f4107f 100644 --- a/process/logsevents/serialize.go +++ b/process/logsevents/serialize.go @@ -11,7 +11,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" ) -// SerializeLogs will serialize the provided logs in a way that Elastic Search expects a bulk request +// SerializeLogs will serialize the provided logs in a way that Elasticsearch expects a bulk request func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data.BufferSlice, index string) error { for _, lg := range logs { meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(lg.ID), "\n")) @@ -50,7 +50,7 @@ func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data. return nil } -// SerializeSCDeploys will serialize the provided smart contract deploys in a way that Elastic Search expects a bulk request +// SerializeSCDeploys will serialize the provided smart contract deploys in a way that Elasticsearch expects a bulk request func (*logsAndEventsProcessor) SerializeSCDeploys(deploys map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice, index string) error { for scAddr, deployInfo := range deploys { meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(scAddr), "\n")) diff --git a/process/miniblocks/serialize.go b/process/miniblocks/serialize.go index 0a436a8a..185fdbe7 100644 --- a/process/miniblocks/serialize.go +++ b/process/miniblocks/serialize.go @@ -9,7 +9,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/block" ) -// SerializeBulkMiniBlocks will serialize the provided miniblocks slice in a way that Elastic Search expects a bulk request +// SerializeBulkMiniBlocks will serialize the provided miniblocks slice in a way that Elasticsearch expects a bulk request func (mp *miniblocksProcessor) SerializeBulkMiniBlocks( bulkMbs []*data.Miniblock, existsInDb map[string]bool, diff --git a/process/miniblocks/serialize_test.go b/process/miniblocks/serialize_test.go index a98b6134..c2f09673 100644 --- a/process/miniblocks/serialize_test.go +++ b/process/miniblocks/serialize_test.go @@ -1,11 +1,11 @@ package miniblocks import ( - "github.com/ElrondNetwork/elrond-go-core/data/block" "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/stretchr/testify/require" ) diff --git a/process/tags/serialize.go b/process/tags/serialize.go index d5ab7b8a..503a005a 100644 --- a/process/tags/serialize.go +++ b/process/tags/serialize.go @@ -3,11 +3,12 @@ package tags import ( "encoding/base64" "fmt" + "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" ) -// Serialize will serialize tagsCount in a way that Elastic Search expects a bulk request +// Serialize will serialize tagsCount in a way that Elasticsearch expects a bulk request func (tc *tagsCount) Serialize(buffSlice *data.BufferSlice, index string) error { for tag, count := range tc.tags { if tag == "" { diff --git a/process/tags/serialize_test.go b/process/tags/serialize_test.go index 087b0ae3..c71bc57a 100644 --- a/process/tags/serialize_test.go +++ b/process/tags/serialize_test.go @@ -1,9 +1,9 @@ package tags import ( - "github.com/ElrondNetwork/elastic-indexer-go/data" "testing" + "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/stretchr/testify/require" ) diff --git a/process/transactions/transactionsProcessor_test.go b/process/transactions/transactionsProcessor_test.go index ab68686b..91ee7fd0 100644 --- a/process/transactions/transactionsProcessor_test.go +++ b/process/transactions/transactionsProcessor_test.go @@ -3,7 +3,6 @@ package transactions import ( "encoding/hex" "fmt" - "github.com/ElrondNetwork/elrond-go-core/data/outport" "math/big" "testing" @@ -13,6 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/pubkeyConverter" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/data/receipt" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" diff --git a/tools/clusters-checker/pkg/checkers/interface.go b/tools/clusters-checker/pkg/checkers/interface.go index af6de57d..96e98da4 100644 --- a/tools/clusters-checker/pkg/checkers/interface.go +++ b/tools/clusters-checker/pkg/checkers/interface.go @@ -1,6 +1,6 @@ package checkers -// ESClient defines what a ES client should do +// ESClient defines what an ES client should do type ESClient interface { InitializeScroll(index string, body []byte, response interface{}) (string, bool, error) DoScrollRequestV2(scrollID string, response interface{}) (string, bool, error) diff --git a/tools/index-modifier/pkg/client/client.go b/tools/index-modifier/pkg/client/client.go index 77df7110..8429e0ae 100644 --- a/tools/index-modifier/pkg/client/client.go +++ b/tools/index-modifier/pkg/client/client.go @@ -4,11 +4,11 @@ import ( "bytes" "context" "fmt" - logger "github.com/ElrondNetwork/elrond-go-logger" "io/ioutil" "net/http" "time" + logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/tidwall/gjson" From 1e0468760ab130889e10418bfcc95e4ee6256dd7 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 1 Sep 2022 14:54:41 +0300 Subject: [PATCH 30/96] indexer binary --- cmd/elasticindexer/config/config.toml | 29 ++++++ cmd/elasticindexer/flags.go | 26 ++++++ cmd/elasticindexer/main.go | 127 ++++++++++++++++++++++++++ config/config.go | 39 ++++++++ factory/indexerFactory.go | 5 +- go.mod | 6 +- go.sum | 12 ++- 7 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 cmd/elasticindexer/config/config.toml create mode 100644 cmd/elasticindexer/flags.go create mode 100644 cmd/elasticindexer/main.go create mode 100644 config/config.go diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml new file mode 100644 index 00000000..85b5b9ba --- /dev/null +++ b/cmd/elasticindexer/config/config.toml @@ -0,0 +1,29 @@ +[config] + enabled-indices = [ + "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", + "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "collections" + ] + [config.elastic-cluster] + use-kibana = false + url = "http://localhost:9200" + username = "" + password = "" + bulk-request-max-size-in-bytes = 4194304 # 4MB + [config.web-socket] + server-url = "localhost:22111" + data-marshaller-type = "json" + [config.address-converter] + length = 32 + type = "bech32" + [config.validator-keys-converter] + length = 96 + type = "hex" + [config.hasher] + type = "blake2b" + [config.economics] + denomination = 18 + [config.logs] + log-file-life-span-in-mb = 1024 # 1GB + log-file-life-span-in-sec = 432000 # 5 days + log-file-prefix = "elastic-indexer" + logs-path = "logs" diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go new file mode 100644 index 00000000..5ec81789 --- /dev/null +++ b/cmd/elasticindexer/flags.go @@ -0,0 +1,26 @@ +package main + +import ( + logger "github.com/ElrondNetwork/elrond-go-logger" + "github.com/urfave/cli" +) + +var ( + configurationFile = cli.StringFlag{ + Name: "config", + Usage: "The main configuration file to load", + Value: "./config/config.toml", + } + logLevel = cli.StringFlag{ + Name: "log-level", + Usage: "This flag specifies the logger `level(s)`. It can contain multiple comma-separated value. For example" + + ", if set to *:INFO the logs for all packages will have the INFO level. However, if set to *:INFO,api:DEBUG" + + " the logs for all packages will have the INFO level, excepting the api package which will receive a DEBUG" + + " log level.", + Value: "*:" + logger.LogInfo.String(), + } + logSaveFile = cli.BoolFlag{ + Name: "log-save", + Usage: "Boolean option for enabling log saving. If set, it will automatically save all the logs into a file.", + } +) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go new file mode 100644 index 00000000..3f70aeb9 --- /dev/null +++ b/cmd/elasticindexer/main.go @@ -0,0 +1,127 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/ElrondNetwork/elastic-indexer-go/config" + "github.com/ElrondNetwork/elrond-go-core/core" + "github.com/ElrondNetwork/elrond-go-core/core/check" + logger "github.com/ElrondNetwork/elrond-go-logger" + "github.com/ElrondNetwork/elrond-go-logger/file" + "github.com/urfave/cli" +) + +var ( + log = logger.GetOrCreate("indexer") + helpTemplate = `NAME: + {{.Name}} - {{.Usage}} +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}} + {{if len .Authors}} +AUTHOR: + {{range .Authors}}{{ . }}{{end}} + {{end}}{{if .Commands}} +GLOBAL OPTIONS: + {{range .VisibleFlags}}{{.}} + {{end}} +VERSION: + {{.Version}} + {{end}} +` +) + +func main() { + app := cli.NewApp() + cli.AppHelpTemplate = helpTemplate + app.Name = "Elastic indexer" + app.Usage = "This tool will index data in an Elasticsearch database" + app.Flags = []cli.Flag{ + configurationFile, + logLevel, + logSaveFile, + } + app.Authors = []cli.Author{ + { + Name: "The Elrond Team", + Email: "contact@elrond.com", + }, + } + + app.Action = startIndexer + + err := app.Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +func startIndexer(ctx *cli.Context) error { + configurationFileName := ctx.GlobalString(configurationFile.Name) + cfg, err := loadMainConfig(configurationFileName) + if err != nil { + return err + } + + fileLogging, err := initializeLogger(ctx, cfg) + if err != nil { + return err + } + + log.Info("closing indexer") + if !check.IfNil(fileLogging) { + err = fileLogging.Close() + log.LogIfError(err) + } + + return nil +} + +func loadMainConfig(filepath string) (*config.Config, error) { + cfg := &config.Config{} + err := core.LoadTomlFile(cfg, filepath) + if err != nil { + return nil, err + } + return cfg, nil +} + +func initializeLogger(ctx *cli.Context, cfg *config.Config) (file.FileLoggingHandler, error) { + logLevelFlagValue := ctx.GlobalString(logLevel.Name) + err := logger.SetLogLevel(logLevelFlagValue) + if err != nil { + return nil, err + } + + withLogFile := ctx.GlobalBool(logSaveFile.Name) + if !withLogFile { + return nil, nil + } + + workingDir, err := os.Getwd() + if err != nil { + log.LogIfError(err) + workingDir = "" + } + + fileLogging, err := file.NewFileLogging(file.ArgsFileLogging{ + WorkingDir: workingDir, + DefaultLogsPath: cfg.Config.Logs.LogsPath, + LogFilePrefix: cfg.Config.Logs.LogFilePrefix, + }) + if err != nil { + return nil, fmt.Errorf("%w creating a log file", err) + } + + err = fileLogging.ChangeFileLifeSpan( + time.Second*time.Duration(cfg.Config.Logs.LogFileLifeSpanInSec), + uint64(cfg.Config.Logs.LogFileLifeSpanInMB), + ) + if err != nil { + return nil, err + } + + return fileLogging, nil +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000..e635646b --- /dev/null +++ b/config/config.go @@ -0,0 +1,39 @@ +package config + +// Config will hold the whole config file's data +type Config struct { + Config struct { + EnabledIndices []string `toml:"enabled-indices"` + ElasticCluster struct { + UseKibana bool `toml:"use-kibana"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` + BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + } `toml:"elastic-cluster"` + WebSocket struct { + ServerURL string `toml:"server-url"` + DataMarshallerTyper string `toml:"data-marshaller-type"` + } `toml:"web-socket"` + AddressConverter struct { + Length int `toml:"length"` + Type string `toml:"type"` + } `toml:"address-converter"` + ValidatorKeysConverter struct { + Length int `toml:"length"` + Type string `toml:"type"` + } `toml:"validator-keys-converter"` + Hasher struct { + Type string `toml:"type"` + } `toml:"hasher"` + Economics struct { + Denomination int `toml:"denomination"` + } `toml:"economics"` + Logs struct { + LogFileLifeSpanInMB int `toml:"log-file-life-span-in-mb"` + LogFileLifeSpanInSec int `toml:"log-file-life-span-in-sec"` + LogFilePrefix string `toml:"log-file-prefix"` + LogsPath string `toml:"logs-path"` + } `toml:"logs"` + } `toml:"config"` +} diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index 44872178..7cdfd047 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -23,8 +23,9 @@ var log = logger.GetOrCreate("indexer/factory") // ArgsIndexerFactory holds all dependencies required by the data indexer factory in order to create // new instances type ArgsIndexerFactory struct { - Enabled bool - UseKibana bool + Enabled bool + UseKibana bool + // move this in ArgSaveBlock IsInImportDBMode bool IndexerCacheSize int Denomination int diff --git a/go.mod b/go.mod index 3fcd2765..1971c3e9 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,17 @@ go 1.17 require ( github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00 - github.com/ElrondNetwork/elrond-go-logger v1.0.7 + github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 + github.com/urfave/cli v1.22.9 ) require ( github.com/btcsuite/btcutil v1.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -20,6 +22,8 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect diff --git a/go.sum b/go.sum index 30b474dd..f510b9aa 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,12 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00 h1:pH+/JIebRGRq5PaC+qZZ7ZBr+YWDDI/N+hZtCD4onUo= github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= -github.com/ElrondNetwork/elrond-go-logger v1.0.7 h1:Ldl1rVS0RGKc1IsW8jIaGCb6Zwei04gsMvyjL05X6mE= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= +github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= +github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= github.com/ElrondNetwork/elrond-vm-common v1.3.7 h1:nd3pTbZD+JQfhCYX4n2QlysGbIsllGjdMU4BuFfiyUk= github.com/ElrondNetwork/elrond-vm-common v1.3.7/go.mod h1:seROQuR7RJCoCS7mgRXVAlvjztltY1c+UroAgWr/USE= @@ -20,6 +22,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -59,8 +63,12 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -72,6 +80,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw= +github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From e5faae51abbf138a48a3b1e9edb2968cda42b300 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 1 Sep 2022 16:40:17 +0300 Subject: [PATCH 31/96] remove shard coordinator from dataIndexer.go --- dataIndexer.go | 4 ---- dataIndexer_test.go | 1 - factory/indexerFactory.go | 1 - 3 files changed, 6 deletions(-) diff --git a/dataIndexer.go b/dataIndexer.go index 97aac835..a98fa037 100644 --- a/dataIndexer.go +++ b/dataIndexer.go @@ -11,7 +11,6 @@ import ( // ArgDataIndexer is a structure that is used to store all the components that are needed to create an indexer type ArgDataIndexer struct { - ShardCoordinator ShardCoordinator Marshalizer marshal.Marshalizer DataDispatcher DispatcherHandler ElasticProcessor ElasticProcessor @@ -51,9 +50,6 @@ func checkIndexerArgs(arguments ArgDataIndexer) error { if check.IfNil(arguments.Marshalizer) { return ErrNilMarshalizer } - if check.IfNil(arguments.ShardCoordinator) { - return ErrNilShardCoordinator - } return nil } diff --git a/dataIndexer_test.go b/dataIndexer_test.go index 99105f30..e9838248 100644 --- a/dataIndexer_test.go +++ b/dataIndexer_test.go @@ -17,7 +17,6 @@ func NewDataIndexerArguments() ArgDataIndexer { Marshalizer: &mock.MarshalizerMock{}, DataDispatcher: &mock.DispatcherMock{}, ElasticProcessor: &mock.ElasticProcessorStub{}, - ShardCoordinator: &mock.ShardCoordinatorMock{}, } } diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index 7cdfd047..030e16e4 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -67,7 +67,6 @@ func NewIndexer(args *ArgsIndexerFactory) (indexer.Indexer, error) { arguments := indexer.ArgDataIndexer{ Marshalizer: args.Marshalizer, - ShardCoordinator: args.ShardCoordinator, ElasticProcessor: elasticProcessor, DataDispatcher: dispatcher, } From 7356166a22114e657cd734c50bc7290a18467257 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 1 Sep 2022 17:00:48 +0300 Subject: [PATCH 32/96] change folder structure --- client/elasticClient.go | 6 +- client/elasticClientCommon.go | 8 +-- client/elasticClientCommon_test.go | 6 +- client/elasticClient_test.go | 2 +- data/logs.go | 2 +- factory/indexerFactory.go | 30 +++++----- factory/indexerFactory_test.go | 18 +++--- .../accountsBalanceNftTransfer_test.go | 2 +- .../accountsBalanceWithLowerTimestamp_test.go | 2 +- integrationtests/accountsESDTRollback_test.go | 2 +- .../accountsESDTWithTokenType_test.go | 2 +- integrationtests/claimRewards_test.go | 2 +- integrationtests/collectionsIndex_test.go | 2 +- integrationtests/createNFTWithTags_test.go | 2 +- integrationtests/esdtTransfer_test.go | 2 +- .../issueTokenAndSetRoles_test.go | 2 +- integrationtests/issueToken_test.go | 2 +- integrationtests/logsCrossShard_test.go | 2 +- integrationtests/nftIssueCreateBurn_test.go | 2 +- .../nftTransferCrossShard_test.go | 2 +- .../nftTransferCrossWithScCall_test.go | 2 +- integrationtests/relayedTx_test.go | 2 +- integrationtests/scCallIntraShard_test.go | 2 +- integrationtests/transactions_test.go | 2 +- integrationtests/updateNFT_test.go | 2 +- integrationtests/utils.go | 18 +++--- .../dataindexer/constants.go | 2 +- .../dataindexer/dataDispatcher.go | 2 +- .../dataindexer/dataDispatcher_test.go | 2 +- .../dataindexer/dataIndexer.go | 2 +- .../dataindexer/dataIndexer_test.go | 2 +- errors.go => process/dataindexer/errors.go | 2 +- .../dataindexer/interface.go | 2 +- .../dataindexer/nilIndexer.go | 2 +- .../accounts/accountsProcessor.go | 12 ++-- .../accounts/accountsProcessor_test.go | 52 ++++++++-------- process/{ => elasticproc}/accounts/errors.go | 0 .../{ => elasticproc}/accounts/serialize.go | 2 +- .../accounts/serialize_test.go | 2 +- .../{ => elasticproc}/block/blockProcessor.go | 4 +- .../block/blockProcessor_test.go | 6 +- process/{ => elasticproc}/block/serialize.go | 10 ++-- .../{ => elasticproc}/block/serialize_test.go | 8 +-- process/{ => elasticproc}/check.go | 4 +- .../collections/serialize.go | 2 +- .../elasticproc/converters}/balanceToFloat.go | 2 +- .../converters}/balanceToFloat_test.go | 0 .../elasticproc/converters}/code.go | 0 .../elasticproc/converters}/events.go | 0 .../elasticproc/converters}/events_test.go | 0 .../elasticproc/converters}/json.go | 0 .../elasticproc/converters}/json_test.go | 0 .../elasticproc/converters}/tags.go | 0 .../elasticproc/converters}/tags_test.go | 0 .../converters}/tokenIdentifier.go | 0 .../converters}/tokenIdentifier_test.go | 0 .../elasticproc/converters}/tokenMetaData.go | 0 .../converters}/tokenMetaData_test.go | 0 process/{ => elasticproc}/elasticProcessor.go | 12 ++-- .../elasticProcessor_test.go | 60 +++++++++---------- process/{ => elasticproc}/interface.go | 4 +- .../logsevents/delegatorsProcessor.go | 2 +- .../logsevents/delegatorsProcessor_test.go | 2 +- .../logsevents/esdtIssueProcessor.go | 0 .../logsevents/esdtIssueProcessor_test.go | 0 .../logsevents/esdtPropertiesProcessor.go | 0 .../esdtPropertiesProcessor_test.go | 2 +- .../logsevents/fungibleESDTProcessor.go | 6 +- .../logsevents/fungibleESDTProcessor_test.go | 0 .../logsevents/informativeLogsProcessor.go | 0 .../informativeLogsProcessor_test.go | 0 .../{ => elasticproc}/logsevents/interface.go | 2 +- .../logsevents/logsAndEventsProcessor.go | 16 ++--- .../logsevents/logsAndEventsProcessor_test.go | 4 +- .../{ => elasticproc}/logsevents/logsData.go | 4 +- .../logsevents/nftsProcessor.go | 12 ++-- .../logsevents/nftsProcessor_test.go | 0 .../logsevents/nftsPropertiesProcessor.go | 2 +- .../nftsPropertiesProcessor_test.go | 0 .../{ => elasticproc}/logsevents/scDeploys.go | 0 .../logsevents/scDeploys_test.go | 0 .../{ => elasticproc}/logsevents/serialize.go | 50 ++++++++-------- .../logsevents/serialize_test.go | 0 .../miniblocks/miniblocksProcessor.go | 6 +- .../miniblocks/miniblocksProcessor_test.go | 6 +- .../{ => elasticproc}/miniblocks/serialize.go | 2 +- .../miniblocks/serialize_test.go | 0 .../operations/operationsProcessor.go | 8 +-- .../operations/operationsProcessor_test.go | 2 +- .../{ => elasticproc}/operations/serialize.go | 2 +- .../operations/serialize_test.go | 0 .../{ => elasticproc}/statistics/serialize.go | 0 .../statistics/serialize_test.go | 0 process/{ => elasticproc}/tags/serialize.go | 6 +- .../{ => elasticproc}/tags/serialize_test.go | 0 process/{ => elasticproc}/tags/tagsCount.go | 0 .../{ => elasticproc}/tags/tagsCount_test.go | 0 .../templatesAndPolicies/interface.go | 0 .../templatesAndPolicies/noKibana.go | 2 +- .../templatesAndPolicies/noKibana_test.go | 0 .../templatesAndPoliciesFactory.go | 0 .../templatesAndPoliciesFactory_test.go | 0 .../templatesAndPolicies/withKibana.go | 2 +- .../templatesAndPolicies/withKibana_test.go | 0 .../{ => elasticproc}/tokeninfo/tokeninfo.go | 0 .../tokeninfo/tokeninfo_test.go | 0 .../transactions/checkers.go | 2 +- .../transactions/checkers_test.go | 2 +- .../transactions/interface.go | 0 .../transactions/scrsDataToTransactions.go | 0 .../scrsDataToTransactions_test.go | 0 .../transactions/serialize.go | 18 +++--- .../transactions/serialize_test.go | 0 .../smartContractResultsProcessor.go | 2 +- .../smartContractResultsProcessor_test.go | 0 .../transactions/transactionDBBuilder.go | 2 +- .../transactions/transactionDBBuilder_test.go | 0 .../transactions/transactionsGrouper.go | 0 .../transactions/transactionsGrouper_test.go | 0 .../transactions/transactionsProcessor.go | 2 +- .../transactionsProcessor_test.go | 0 process/{ => elasticproc}/updateTokenType.go | 4 +- .../{ => elasticproc}/validators/serialize.go | 0 .../validators/serialize_test.go | 0 .../validators/validatorsProcessor.go | 4 +- .../validators/validatorsProcessor_test.go | 4 +- process/factory/elasticProcessorFactory.go | 36 +++++------ 127 files changed, 268 insertions(+), 268 deletions(-) rename constants.go => process/dataindexer/constants.go (99%) rename dataDispatcher.go => process/dataindexer/dataDispatcher.go (99%) rename dataDispatcher_test.go => process/dataindexer/dataDispatcher_test.go (99%) rename dataIndexer.go => process/dataindexer/dataIndexer.go (99%) rename dataIndexer_test.go => process/dataindexer/dataIndexer_test.go (99%) rename errors.go => process/dataindexer/errors.go (99%) rename interface.go => process/dataindexer/interface.go (99%) rename nilIndexer.go => process/dataindexer/nilIndexer.go (98%) rename process/{ => elasticproc}/accounts/accountsProcessor.go (96%) rename process/{ => elasticproc}/accounts/accountsProcessor_test.go (91%) rename process/{ => elasticproc}/accounts/errors.go (100%) rename process/{ => elasticproc}/accounts/serialize.go (98%) rename process/{ => elasticproc}/accounts/serialize_test.go (99%) rename process/{ => elasticproc}/block/blockProcessor.go (98%) rename process/{ => elasticproc}/block/blockProcessor_test.go (95%) rename process/{ => elasticproc}/block/serialize.go (85%) rename process/{ => elasticproc}/block/serialize_test.go (94%) rename process/{ => elasticproc}/check.go (92%) rename process/{ => elasticproc}/collections/serialize.go (96%) rename {converters => process/elasticproc/converters}/balanceToFloat.go (96%) rename {converters => process/elasticproc/converters}/balanceToFloat_test.go (100%) rename {converters => process/elasticproc/converters}/code.go (100%) rename {converters => process/elasticproc/converters}/events.go (100%) rename {converters => process/elasticproc/converters}/events_test.go (100%) rename {converters => process/elasticproc/converters}/json.go (100%) rename {converters => process/elasticproc/converters}/json_test.go (100%) rename {converters => process/elasticproc/converters}/tags.go (100%) rename {converters => process/elasticproc/converters}/tags_test.go (100%) rename {converters => process/elasticproc/converters}/tokenIdentifier.go (100%) rename {converters => process/elasticproc/converters}/tokenIdentifier_test.go (100%) rename {converters => process/elasticproc/converters}/tokenMetaData.go (100%) rename {converters => process/elasticproc/converters}/tokenMetaData_test.go (100%) rename process/{ => elasticproc}/elasticProcessor.go (98%) rename process/{ => elasticproc}/elasticProcessor_test.go (90%) rename process/{ => elasticproc}/interface.go (98%) rename process/{ => elasticproc}/logsevents/delegatorsProcessor.go (97%) rename process/{ => elasticproc}/logsevents/delegatorsProcessor_test.go (97%) rename process/{ => elasticproc}/logsevents/esdtIssueProcessor.go (100%) rename process/{ => elasticproc}/logsevents/esdtIssueProcessor_test.go (100%) rename process/{ => elasticproc}/logsevents/esdtPropertiesProcessor.go (100%) rename process/{ => elasticproc}/logsevents/esdtPropertiesProcessor_test.go (97%) rename process/{ => elasticproc}/logsevents/fungibleESDTProcessor.go (93%) rename process/{ => elasticproc}/logsevents/fungibleESDTProcessor_test.go (100%) rename process/{ => elasticproc}/logsevents/informativeLogsProcessor.go (100%) rename process/{ => elasticproc}/logsevents/informativeLogsProcessor_test.go (100%) rename process/{ => elasticproc}/logsevents/interface.go (92%) rename process/{ => elasticproc}/logsevents/logsAndEventsProcessor.go (94%) rename process/{ => elasticproc}/logsevents/logsAndEventsProcessor_test.go (98%) rename process/{ => elasticproc}/logsevents/logsData.go (89%) rename process/{ => elasticproc}/logsevents/nftsProcessor.go (91%) rename process/{ => elasticproc}/logsevents/nftsProcessor_test.go (100%) rename process/{ => elasticproc}/logsevents/nftsPropertiesProcessor.go (96%) rename process/{ => elasticproc}/logsevents/nftsPropertiesProcessor_test.go (100%) rename process/{ => elasticproc}/logsevents/scDeploys.go (100%) rename process/{ => elasticproc}/logsevents/scDeploys_test.go (100%) rename process/{ => elasticproc}/logsevents/serialize.go (84%) rename process/{ => elasticproc}/logsevents/serialize_test.go (100%) rename process/{ => elasticproc}/miniblocks/miniblocksProcessor.go (97%) rename process/{ => elasticproc}/miniblocks/miniblocksProcessor_test.go (96%) rename process/{ => elasticproc}/miniblocks/serialize.go (96%) rename process/{ => elasticproc}/miniblocks/serialize_test.go (100%) rename process/{ => elasticproc}/operations/operationsProcessor.go (92%) rename process/{ => elasticproc}/operations/operationsProcessor_test.go (96%) rename process/{ => elasticproc}/operations/serialize.go (94%) rename process/{ => elasticproc}/operations/serialize_test.go (100%) rename process/{ => elasticproc}/statistics/serialize.go (100%) rename process/{ => elasticproc}/statistics/serialize_test.go (100%) rename process/{ => elasticproc}/tags/serialize.go (75%) rename process/{ => elasticproc}/tags/serialize_test.go (100%) rename process/{ => elasticproc}/tags/tagsCount.go (100%) rename process/{ => elasticproc}/tags/tagsCount_test.go (100%) rename process/{ => elasticproc}/templatesAndPolicies/interface.go (100%) rename process/{ => elasticproc}/templatesAndPolicies/noKibana.go (96%) rename process/{ => elasticproc}/templatesAndPolicies/noKibana_test.go (100%) rename process/{ => elasticproc}/templatesAndPolicies/templatesAndPoliciesFactory.go (100%) rename process/{ => elasticproc}/templatesAndPolicies/templatesAndPoliciesFactory_test.go (100%) rename process/{ => elasticproc}/templatesAndPolicies/withKibana.go (97%) rename process/{ => elasticproc}/templatesAndPolicies/withKibana_test.go (100%) rename process/{ => elasticproc}/tokeninfo/tokeninfo.go (100%) rename process/{ => elasticproc}/tokeninfo/tokeninfo_test.go (100%) rename process/{ => elasticproc}/transactions/checkers.go (97%) rename process/{ => elasticproc}/transactions/checkers_test.go (98%) rename process/{ => elasticproc}/transactions/interface.go (100%) rename process/{ => elasticproc}/transactions/scrsDataToTransactions.go (100%) rename process/{ => elasticproc}/transactions/scrsDataToTransactions_test.go (100%) rename process/{ => elasticproc}/transactions/serialize.go (88%) rename process/{ => elasticproc}/transactions/serialize_test.go (100%) rename process/{ => elasticproc}/transactions/smartContractResultsProcessor.go (98%) rename process/{ => elasticproc}/transactions/smartContractResultsProcessor_test.go (100%) rename process/{ => elasticproc}/transactions/transactionDBBuilder.go (98%) rename process/{ => elasticproc}/transactions/transactionDBBuilder_test.go (100%) rename process/{ => elasticproc}/transactions/transactionsGrouper.go (100%) rename process/{ => elasticproc}/transactions/transactionsGrouper_test.go (100%) rename process/{ => elasticproc}/transactions/transactionsProcessor.go (99%) rename process/{ => elasticproc}/transactions/transactionsProcessor_test.go (100%) rename process/{ => elasticproc}/updateTokenType.go (95%) rename process/{ => elasticproc}/validators/serialize.go (100%) rename process/{ => elasticproc}/validators/serialize_test.go (100%) rename process/{ => elasticproc}/validators/validatorsProcessor.go (91%) rename process/{ => elasticproc}/validators/validatorsProcessor_test.go (84%) diff --git a/client/elasticClient.go b/client/elasticClient.go index 236e97f2..f1e18dbc 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -8,8 +8,8 @@ import ( "net/http" "strings" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" @@ -41,7 +41,7 @@ type elasticClient struct { // NewElasticClient will create a new instance of elasticClient func NewElasticClient(cfg elasticsearch.Config) (*elasticClient, error) { if len(cfg.Addresses) == 0 { - return nil, indexer.ErrNoElasticUrlProvided + return nil, dataindexer.ErrNoElasticUrlProvided } es, err := elasticsearch.NewClient(cfg) @@ -302,7 +302,7 @@ func (ec *elasticClient) createPolicy(policyName string, policy *bytes.Buffer) e errStr := fmt.Sprintf("%v", existsRes.Error) if existsRes.Status == http.StatusConflict && !strings.Contains(errStr, errPolicyAlreadyExists) { - return indexer.ErrCouldNotCreatePolicy + return dataindexer.ErrCouldNotCreatePolicy } return nil diff --git a/client/elasticClientCommon.go b/client/elasticClientCommon.go index 99782d47..19f1bf7a 100644 --- a/client/elasticClientCommon.go +++ b/client/elasticClientCommon.go @@ -10,8 +10,8 @@ import ( "net/url" "strings" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/elastic/go-elasticsearch/v7/esapi" ) @@ -67,7 +67,7 @@ func elasticDefaultErrorResponseHandler(res *esapi.Response) error { isBackOffError := strings.Contains(string(bodyBytes), fmt.Sprintf("%d", http.StatusForbidden)) || strings.Contains(string(bodyBytes), fmt.Sprintf("%d", http.StatusTooManyRequests)) if isBackOffError { - errToReturn = indexer.ErrBackOff + errToReturn = dataindexer.ErrBackOff } return fmt.Errorf("%w, cannot unmarshal elastic response body to map[string]interface{}, "+ @@ -213,7 +213,7 @@ func kibanaResponseErrorHandler(res *esapi.Response) error { log.Warn("elasticClient.parseResponse", "error returned by elastic API", errorRes.Error, "code", res.StatusCode) - return indexer.ErrBackOff + return dataindexer.ErrBackOff } func newRequest(method, path string, body *bytes.Buffer) *http.Request { @@ -263,7 +263,7 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE if err != nil { log.Warn("elasticClient.parseResponse", "could not load response body:", err.Error()) - return indexer.ErrBackOff + return dataindexer.ErrBackOff } return nil diff --git a/client/elasticClientCommon_test.go b/client/elasticClientCommon_test.go index 10442d11..376e01b1 100644 --- a/client/elasticClientCommon_test.go +++ b/client/elasticClientCommon_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -55,7 +55,7 @@ func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpForbiddenError resp := createMockEsapiResponseWithText(httpErrString) err := elasticDefaultErrorResponseHandler(resp) - assert.True(t, errorsGo.Is(err, indexer.ErrBackOff)) + assert.True(t, errorsGo.Is(err, dataindexer.ErrBackOff)) } func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpTooManyRequestsErrorShouldSignalBackOffErr(t *testing.T) { @@ -65,7 +65,7 @@ func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpTooManyRequest resp := createMockEsapiResponseWithText(httpErrString) err := elasticDefaultErrorResponseHandler(resp) - assert.True(t, errorsGo.Is(err, indexer.ErrBackOff)) + assert.True(t, errorsGo.Is(err, dataindexer.ErrBackOff)) } func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithGenericError(t *testing.T) { diff --git a/client/elasticClient_test.go b/client/elasticClient_test.go index 213a9144..ab32e718 100644 --- a/client/elasticClient_test.go +++ b/client/elasticClient_test.go @@ -7,9 +7,9 @@ import ( "os" "testing" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/client/logging" "github.com/ElrondNetwork/elastic-indexer-go/data" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/elastic/go-elasticsearch/v7" "github.com/stretchr/testify/require" ) diff --git a/data/logs.go b/data/logs.go index 20ab3b39..8db3cc93 100644 --- a/data/logs.go +++ b/data/logs.go @@ -3,7 +3,7 @@ package data import ( "time" - "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" ) // Logs holds all the fields needed for a logs structure diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index 030e16e4..2600e60e 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -6,9 +6,9 @@ import ( "net/http" "time" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/client" "github.com/ElrondNetwork/elastic-indexer-go/client/logging" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elastic-indexer-go/process/factory" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" @@ -35,7 +35,7 @@ type ArgsIndexerFactory struct { Password string TemplatesPath string EnabledIndexes []string - ShardCoordinator indexer.ShardCoordinator + ShardCoordinator dataindexer.ShardCoordinator Marshalizer marshal.Marshalizer Hasher hashing.Hasher AddressPubkeyConverter core.PubkeyConverter @@ -43,14 +43,14 @@ type ArgsIndexerFactory struct { } // NewIndexer will create a new instance of Indexer -func NewIndexer(args *ArgsIndexerFactory) (indexer.Indexer, error) { +func NewIndexer(args *ArgsIndexerFactory) (dataindexer.Indexer, error) { err := checkDataIndexerParams(args) if err != nil { return nil, err } if !args.Enabled { - return indexer.NewNilIndexer(), nil + return dataindexer.NewNilIndexer(), nil } elasticProcessor, err := createElasticProcessor(args) @@ -58,20 +58,20 @@ func NewIndexer(args *ArgsIndexerFactory) (indexer.Indexer, error) { return nil, err } - dispatcher, err := indexer.NewDataDispatcher(args.IndexerCacheSize) + dispatcher, err := dataindexer.NewDataDispatcher(args.IndexerCacheSize) if err != nil { return nil, err } dispatcher.StartIndexData() - arguments := indexer.ArgDataIndexer{ + arguments := dataindexer.ArgDataIndexer{ Marshalizer: args.Marshalizer, ElasticProcessor: elasticProcessor, DataDispatcher: dispatcher, } - return indexer.NewDataIndexer(arguments) + return dataindexer.NewDataIndexer(arguments) } func retryBackOff(attempt int) time.Duration { @@ -81,7 +81,7 @@ func retryBackOff(attempt int) time.Duration { return d } -func createElasticProcessor(args *ArgsIndexerFactory) (indexer.ElasticProcessor, error) { +func createElasticProcessor(args *ArgsIndexerFactory) (dataindexer.ElasticProcessor, error) { databaseClient, err := client.NewElasticClient(elasticsearch.Config{ Addresses: []string{args.Url}, Username: args.UserName, @@ -113,25 +113,25 @@ func createElasticProcessor(args *ArgsIndexerFactory) (indexer.ElasticProcessor, func checkDataIndexerParams(arguments *ArgsIndexerFactory) error { if arguments.IndexerCacheSize < 0 { - return indexer.ErrNegativeCacheSize + return dataindexer.ErrNegativeCacheSize } if check.IfNil(arguments.AddressPubkeyConverter) { - return fmt.Errorf("%w when setting AddressPubkeyConverter in indexer", indexer.ErrNilPubkeyConverter) + return fmt.Errorf("%w when setting AddressPubkeyConverter in indexer", dataindexer.ErrNilPubkeyConverter) } if check.IfNil(arguments.ValidatorPubkeyConverter) { - return fmt.Errorf("%w when setting ValidatorPubkeyConverter in indexer", indexer.ErrNilPubkeyConverter) + return fmt.Errorf("%w when setting ValidatorPubkeyConverter in indexer", dataindexer.ErrNilPubkeyConverter) } if arguments.Url == "" { - return indexer.ErrNilUrl + return dataindexer.ErrNilUrl } if check.IfNil(arguments.Marshalizer) { - return indexer.ErrNilMarshalizer + return dataindexer.ErrNilMarshalizer } if check.IfNil(arguments.Hasher) { - return indexer.ErrNilHasher + return dataindexer.ErrNilHasher } if check.IfNil(arguments.ShardCoordinator) { - return indexer.ErrNilShardCoordinator + return dataindexer.ErrNilShardCoordinator } return nil diff --git a/factory/indexerFactory_test.go b/factory/indexerFactory_test.go index 96d72f3f..4aa66c0a 100644 --- a/factory/indexerFactory_test.go +++ b/factory/indexerFactory_test.go @@ -6,8 +6,8 @@ import ( "net/http/httptest" "testing" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/stretchr/testify/require" ) @@ -44,7 +44,7 @@ func TestNewIndexerFactory(t *testing.T) { args.IndexerCacheSize = -1 return args }, - exError: indexer.ErrNegativeCacheSize, + exError: dataindexer.ErrNegativeCacheSize, }, { name: "NilAddressPubkeyConverter", @@ -53,7 +53,7 @@ func TestNewIndexerFactory(t *testing.T) { args.AddressPubkeyConverter = nil return args }, - exError: indexer.ErrNilPubkeyConverter, + exError: dataindexer.ErrNilPubkeyConverter, }, { name: "NilValidatorPubkeyConverter", @@ -62,7 +62,7 @@ func TestNewIndexerFactory(t *testing.T) { args.ValidatorPubkeyConverter = nil return args }, - exError: indexer.ErrNilPubkeyConverter, + exError: dataindexer.ErrNilPubkeyConverter, }, { name: "NilMarshalizer", @@ -71,7 +71,7 @@ func TestNewIndexerFactory(t *testing.T) { args.Marshalizer = nil return args }, - exError: indexer.ErrNilMarshalizer, + exError: dataindexer.ErrNilMarshalizer, }, { name: "NilHasher", @@ -80,7 +80,7 @@ func TestNewIndexerFactory(t *testing.T) { args.Hasher = nil return args }, - exError: indexer.ErrNilHasher, + exError: dataindexer.ErrNilHasher, }, { name: "EmptyUrl", @@ -89,7 +89,7 @@ func TestNewIndexerFactory(t *testing.T) { args.Url = "" return args }, - exError: indexer.ErrNilUrl, + exError: dataindexer.ErrNilUrl, }, { name: "NilShardCoordinator", @@ -98,7 +98,7 @@ func TestNewIndexerFactory(t *testing.T) { args.ShardCoordinator = nil return args }, - exError: indexer.ErrNilShardCoordinator, + exError: dataindexer.ErrNilShardCoordinator, }, { name: "All arguments ok", @@ -125,7 +125,7 @@ func TestIndexerFactoryCreate_NilIndexer(t *testing.T) { nilIndexer, err := NewIndexer(args) require.NoError(t, err) - _, ok := nilIndexer.(*indexer.NilIndexer) + _, ok := nilIndexer.(*dataindexer.NilIndexer) require.True(t, ok) } diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index a35c974b..0e393b1d 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 74327bec..99b2811c 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 46cbc747..35fe4168 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -8,8 +8,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index e7f409aa..4229098b 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index 6349b963..d5bb9081 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index 46499d85..dbd1b4f6 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -8,8 +8,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 29b1f806..125f2910 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -8,8 +8,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 19f248fa..a197f0b6 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 92fe4b7e..8cc168ea 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -6,8 +6,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 0128ea14..418de4be 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -6,8 +6,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index fb2e6a4c..15d01b66 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index ddd01f4c..f0655722 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 51508854..882236bf 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 70b4448d..bf087ec3 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 1e746190..1df731fe 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index ad023aba..97ecf9cc 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerData "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerData "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 83781598..4589954c 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerData "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerData "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index 73be4843..b502ff4a 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -7,8 +7,8 @@ import ( "math/big" "testing" - indexerdata "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 227d3c2e..a32aa5d2 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "os" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/client" "github.com/ElrondNetwork/elastic-indexer-go/client/logging" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/process" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc" "github.com/ElrondNetwork/elastic-indexer-go/process/factory" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/elastic/go-elasticsearch/v7" @@ -21,7 +21,7 @@ func setLogLevelDebug() { _ = logger.SetLogLevel("process:DEBUG") } -func createESClient(url string) (process.DatabaseClientHandler, error) { +func createESClient(url string) (elasticproc.DatabaseClientHandler, error) { return client.NewElasticClient(elasticsearch.Config{ Addresses: []string{url}, Logger: &logging.CustomLogger{}, @@ -30,9 +30,9 @@ func createESClient(url string) (process.DatabaseClientHandler, error) { // CreateElasticProcessor - func CreateElasticProcessor( - esClient process.DatabaseClientHandler, - shardCoordinator indexer.ShardCoordinator, -) (indexer.ElasticProcessor, error) { + esClient elasticproc.DatabaseClientHandler, + shardCoordinator dataindexer.ShardCoordinator, +) (dataindexer.ElasticProcessor, error) { args := factory.ArgElasticProcessorFactory{ Marshalizer: &mock.MarshalizerMock{}, Hasher: &mock.HasherMock{}, @@ -40,9 +40,9 @@ func CreateElasticProcessor( ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32), DBClient: esClient, ShardCoordinator: shardCoordinator, - EnabledIndexes: []string{indexer.TransactionsIndex, indexer.LogsIndex, indexer.AccountsESDTIndex, indexer.ScResultsIndex, - indexer.ReceiptsIndex, indexer.BlockIndex, indexer.AccountsIndex, indexer.TokensIndex, indexer.TagsIndex, indexer.CollectionsIndex, - indexer.OperationsIndex}, + EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, + dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.CollectionsIndex, + dataindexer.OperationsIndex}, Denomination: 18, IsInImportDBMode: false, } diff --git a/constants.go b/process/dataindexer/constants.go similarity index 99% rename from constants.go rename to process/dataindexer/constants.go index 3b41d7da..7f54698c 100644 --- a/constants.go +++ b/process/dataindexer/constants.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer const ( // IndexSuffix is the suffix for the Elasticsearch indexes diff --git a/dataDispatcher.go b/process/dataindexer/dataDispatcher.go similarity index 99% rename from dataDispatcher.go rename to process/dataindexer/dataDispatcher.go index f9d4ccc3..ccf605ea 100644 --- a/dataDispatcher.go +++ b/process/dataindexer/dataDispatcher.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer import ( "context" diff --git a/dataDispatcher_test.go b/process/dataindexer/dataDispatcher_test.go similarity index 99% rename from dataDispatcher_test.go rename to process/dataindexer/dataDispatcher_test.go index c46c1176..ace85523 100644 --- a/dataDispatcher_test.go +++ b/process/dataindexer/dataDispatcher_test.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer import ( "context" diff --git a/dataIndexer.go b/process/dataindexer/dataIndexer.go similarity index 99% rename from dataIndexer.go rename to process/dataindexer/dataIndexer.go index a98fa037..57ca283c 100644 --- a/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer import ( "github.com/ElrondNetwork/elastic-indexer-go/data" diff --git a/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go similarity index 99% rename from dataIndexer_test.go rename to process/dataindexer/dataIndexer_test.go index e9838248..cb93b79a 100644 --- a/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer import ( "testing" diff --git a/errors.go b/process/dataindexer/errors.go similarity index 99% rename from errors.go rename to process/dataindexer/errors.go index 6c2e539b..87ebdf48 100644 --- a/errors.go +++ b/process/dataindexer/errors.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer import "errors" diff --git a/interface.go b/process/dataindexer/interface.go similarity index 99% rename from interface.go rename to process/dataindexer/interface.go index 035644af..f9f0313c 100644 --- a/interface.go +++ b/process/dataindexer/interface.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer import ( "math/big" diff --git a/nilIndexer.go b/process/dataindexer/nilIndexer.go similarity index 98% rename from nilIndexer.go rename to process/dataindexer/nilIndexer.go index 000127f2..89d24ff1 100644 --- a/nilIndexer.go +++ b/process/dataindexer/nilIndexer.go @@ -1,4 +1,4 @@ -package indexer +package dataindexer import ( "github.com/ElrondNetwork/elrond-go-core/data" diff --git a/process/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go similarity index 96% rename from process/accounts/accountsProcessor.go rename to process/elasticproc/accounts/accountsProcessor.go index 8368c2d5..d5ffa751 100644 --- a/process/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -6,9 +6,9 @@ import ( "math/big" "time" - indexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data/esdt" @@ -21,21 +21,21 @@ var log = logger.GetOrCreate("indexer/process/accounts") // accountsProcessor is a structure responsible for processing accounts type accountsProcessor struct { addressPubkeyConverter core.PubkeyConverter - balanceConverter indexer.BalanceConverter + balanceConverter dataindexer.BalanceConverter shardID uint32 } // NewAccountsProcessor will create a new instance of accounts processor func NewAccountsProcessor( addressPubkeyConverter core.PubkeyConverter, - balanceConverter indexer.BalanceConverter, + balanceConverter dataindexer.BalanceConverter, shardID uint32, ) (*accountsProcessor, error) { if check.IfNil(addressPubkeyConverter) { - return nil, indexer.ErrNilPubkeyConverter + return nil, dataindexer.ErrNilPubkeyConverter } if check.IfNil(balanceConverter) { - return nil, indexer.ErrNilBalanceConverter + return nil, dataindexer.ErrNilBalanceConverter } return &accountsProcessor{ diff --git a/process/accounts/accountsProcessor_test.go b/process/elasticproc/accounts/accountsProcessor_test.go similarity index 91% rename from process/accounts/accountsProcessor_test.go rename to process/elasticproc/accounts/accountsProcessor_test.go index 01449bbe..4e5c3c46 100644 --- a/process/accounts/accountsProcessor_test.go +++ b/process/elasticproc/accounts/accountsProcessor_test.go @@ -8,11 +8,11 @@ import ( "testing" "time" - indexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/process/tags" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tags" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/outport" @@ -26,27 +26,27 @@ func TestNewAccountsProcessor(t *testing.T) { tests := []struct { name string - argsFunc func() (core.PubkeyConverter, indexer.BalanceConverter, uint32) + argsFunc func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) exError error }{ { name: "NilBalanceConverter", - argsFunc: func() (core.PubkeyConverter, indexer.BalanceConverter, uint32) { - return &mock.PubkeyConverterMock{}, nil,0 + argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) { + return &mock.PubkeyConverterMock{}, nil, 0 }, - exError: indexer.ErrNilBalanceConverter, + exError: dataindexer.ErrNilBalanceConverter, }, { name: "NilPubKeyConverter", - argsFunc: func() (core.PubkeyConverter, indexer.BalanceConverter, uint32) { - return nil, balanceConverter,0 + argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) { + return nil, balanceConverter, 0 }, - exError: indexer.ErrNilPubkeyConverter, + exError: dataindexer.ErrNilPubkeyConverter, }, { name: "ShouldWork", - argsFunc: func() (core.PubkeyConverter, indexer.BalanceConverter, uint32) { - return &mock.PubkeyConverterMock{}, balanceConverter,0 + argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) { + return &mock.PubkeyConverterMock{}, balanceConverter, 0 }, exError: nil, }, @@ -63,7 +63,7 @@ func TestNewAccountsProcessor(t *testing.T) { func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter ,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) regularAccounts, esdtAccounts := ap.GetAccounts(nil, nil) require.Len(t, regularAccounts, 0) @@ -73,7 +73,7 @@ func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { func TestAccountsProcessor_PrepareRegularAccountsMapWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) accountsInfo := ap.PrepareRegularAccountsMap(0, nil) require.Len(t, accountsInfo, 0) @@ -82,7 +82,7 @@ func TestAccountsProcessor_PrepareRegularAccountsMapWithNil(t *testing.T) { func TestGetESDTInfo(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) require.NotNil(t, ap) tokenIdentifier := "token-001" @@ -110,7 +110,7 @@ func TestGetESDTInfo(t *testing.T) { func TestGetESDTInfoNFT(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) require.NotNil(t, ap) tokenIdentifier := "token-001" @@ -142,7 +142,7 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { t.Parallel() pubKeyConverter := mock.NewPubkeyConverterMock(32) - ap, _ := NewAccountsProcessor(pubKeyConverter, balanceConverter,0) + ap, _ := NewAccountsProcessor(pubKeyConverter, balanceConverter, 0) require.NotNil(t, ap) nftName := "Test-nft" @@ -192,7 +192,7 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -221,7 +221,7 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -244,7 +244,7 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -280,7 +280,7 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { IsSender: false, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) require.NotNil(t, ap) res := ap.PrepareRegularAccountsMap(123, []*data.Account{egldAccount}) @@ -325,7 +325,7 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { }, }, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) require.NotNil(t, ap) accountsESDT := []*data.AccountESDT{ @@ -379,7 +379,7 @@ func TestAccountsProcessor_PrepareAccountsHistory(t *testing.T) { }, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) res := ap.PrepareAccountsHistory(100, accounts) accountBalanceHistory := res["addr1-token-112-10"] @@ -400,7 +400,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("no tokens with missing data or nonce higher than 0", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) oldCreator := "old creator" tokensInfo := []*data.TokenInfo{ @@ -416,7 +416,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("error loading token, should not update metadata", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) tokensInfo := []*data.TokenInfo{ { @@ -436,7 +436,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("should work and update metadata", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter,0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) metadata0, metadata1 := &esdt.MetaData{Creator: []byte("creator 0")}, &esdt.MetaData{Creator: []byte("creator 1")} tokensInfo := []*data.TokenInfo{ diff --git a/process/accounts/errors.go b/process/elasticproc/accounts/errors.go similarity index 100% rename from process/accounts/errors.go rename to process/elasticproc/accounts/errors.go diff --git a/process/accounts/serialize.go b/process/elasticproc/accounts/serialize.go similarity index 98% rename from process/accounts/serialize.go rename to process/elasticproc/accounts/serialize.go index 40a466e5..870ede26 100644 --- a/process/accounts/serialize.go +++ b/process/elasticproc/accounts/serialize.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" ) // SerializeNFTCreateInfo will serialize the provided nft create information in a way that Elasticsearch expects a bulk request diff --git a/process/accounts/serialize_test.go b/process/elasticproc/accounts/serialize_test.go similarity index 99% rename from process/accounts/serialize_test.go rename to process/elasticproc/accounts/serialize_test.go index 1d867187..a688c0bf 100644 --- a/process/accounts/serialize_test.go +++ b/process/elasticproc/accounts/serialize_test.go @@ -3,8 +3,8 @@ package accounts import ( "testing" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/stretchr/testify/require" ) diff --git a/process/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go similarity index 98% rename from process/block/blockProcessor.go rename to process/elasticproc/block/blockProcessor.go index bc47cfa6..6f3206df 100644 --- a/process/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -6,9 +6,9 @@ import ( "strconv" "time" - indexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" diff --git a/process/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go similarity index 95% rename from process/block/blockProcessor_test.go rename to process/elasticproc/block/blockProcessor_test.go index fcdf9497..958f4275 100644 --- a/process/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -6,9 +6,9 @@ import ( "math/big" "testing" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" @@ -94,7 +94,7 @@ func TestBlockProcessor_PrepareBlockForDBNilHeader(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"),nil, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0) + dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), nil, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0) require.Equal(t, indexer.ErrNilHeaderHandler, err) require.Nil(t, dbBlock) } @@ -104,7 +104,7 @@ func TestBlockProcessor_PrepareBlockForDBNilBody(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"),&dataBlock.MetaBlock{}, nil, nil, nil, outport.HeaderGasConsumption{}, 0) + dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), &dataBlock.MetaBlock{}, nil, nil, nil, outport.HeaderGasConsumption{}, 0) require.Equal(t, indexer.ErrNilBlockBody, err) require.Nil(t, dbBlock) } diff --git a/process/block/serialize.go b/process/elasticproc/block/serialize.go similarity index 85% rename from process/block/serialize.go rename to process/elasticproc/block/serialize.go index 371bd2e3..56ac1cca 100644 --- a/process/block/serialize.go +++ b/process/elasticproc/block/serialize.go @@ -4,9 +4,9 @@ import ( "encoding/json" "fmt" - "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -15,7 +15,7 @@ import ( // SerializeBlock will serialize a block for database func (bp *blockProcessor) SerializeBlock(elasticBlock *data.Block, buffSlice *data.BufferSlice, index string) error { if elasticBlock == nil { - return indexer.ErrNilElasticBlock + return dataindexer.ErrNilElasticBlock } meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(elasticBlock.Hash), "\n")) @@ -30,12 +30,12 @@ func (bp *blockProcessor) SerializeBlock(elasticBlock *data.Block, buffSlice *da // SerializeEpochInfoData will serialize information about current epoch func (bp *blockProcessor) SerializeEpochInfoData(header coreData.HeaderHandler, buffSlice *data.BufferSlice, index string) error { if check.IfNil(header) { - return indexer.ErrNilHeaderHandler + return dataindexer.ErrNilHeaderHandler } metablock, ok := header.(*block.MetaBlock) if !ok { - return fmt.Errorf("%w in blockProcessor.SerializeEpochInfoData", indexer.ErrHeaderTypeAssertion) + return fmt.Errorf("%w in blockProcessor.SerializeEpochInfoData", dataindexer.ErrHeaderTypeAssertion) } epochInfo := &data.EpochInfo{ diff --git a/process/block/serialize_test.go b/process/elasticproc/block/serialize_test.go similarity index 94% rename from process/block/serialize_test.go rename to process/elasticproc/block/serialize_test.go index a248cbf7..87fccfb5 100644 --- a/process/block/serialize_test.go +++ b/process/elasticproc/block/serialize_test.go @@ -5,9 +5,9 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/stretchr/testify/require" @@ -19,7 +19,7 @@ func TestBlockProcessor_SerializeBlockNilElasticBlockErrors(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) err := bp.SerializeBlock(nil, nil, "") - require.True(t, errors.Is(err, indexer.ErrNilElasticBlock)) + require.True(t, errors.Is(err, dataindexer.ErrNilElasticBlock)) } func TestBlockProcessor_SerializeBlock(t *testing.T) { @@ -41,10 +41,10 @@ func TestBlockProcessor_SerializeEpochInfoDataErrors(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) err := bp.SerializeEpochInfoData(nil, nil, "") - require.Equal(t, indexer.ErrNilHeaderHandler, err) + require.Equal(t, dataindexer.ErrNilHeaderHandler, err) err = bp.SerializeEpochInfoData(&dataBlock.Header{}, nil, "") - require.True(t, errors.Is(err, indexer.ErrHeaderTypeAssertion)) + require.True(t, errors.Is(err, dataindexer.ErrHeaderTypeAssertion)) } func TestBlockProcessor_SerializeEpochInfoData(t *testing.T) { diff --git a/process/check.go b/process/elasticproc/check.go similarity index 92% rename from process/check.go rename to process/elasticproc/check.go index f5965ef8..e9b75032 100644 --- a/process/check.go +++ b/process/elasticproc/check.go @@ -1,7 +1,7 @@ -package process +package elasticproc import ( - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core/check" ) diff --git a/process/collections/serialize.go b/process/elasticproc/collections/serialize.go similarity index 96% rename from process/collections/serialize.go rename to process/elasticproc/collections/serialize.go index 6c68ca98..fbe33274 100644 --- a/process/collections/serialize.go +++ b/process/elasticproc/collections/serialize.go @@ -5,8 +5,8 @@ import ( "fmt" "math/big" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" ) diff --git a/converters/balanceToFloat.go b/process/elasticproc/converters/balanceToFloat.go similarity index 96% rename from converters/balanceToFloat.go rename to process/elasticproc/converters/balanceToFloat.go index fd6f37a1..f4153fd8 100644 --- a/converters/balanceToFloat.go +++ b/process/elasticproc/converters/balanceToFloat.go @@ -4,7 +4,7 @@ import ( "math" "math/big" - indexer "github.com/ElrondNetwork/elastic-indexer-go" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" ) diff --git a/converters/balanceToFloat_test.go b/process/elasticproc/converters/balanceToFloat_test.go similarity index 100% rename from converters/balanceToFloat_test.go rename to process/elasticproc/converters/balanceToFloat_test.go diff --git a/converters/code.go b/process/elasticproc/converters/code.go similarity index 100% rename from converters/code.go rename to process/elasticproc/converters/code.go diff --git a/converters/events.go b/process/elasticproc/converters/events.go similarity index 100% rename from converters/events.go rename to process/elasticproc/converters/events.go diff --git a/converters/events_test.go b/process/elasticproc/converters/events_test.go similarity index 100% rename from converters/events_test.go rename to process/elasticproc/converters/events_test.go diff --git a/converters/json.go b/process/elasticproc/converters/json.go similarity index 100% rename from converters/json.go rename to process/elasticproc/converters/json.go diff --git a/converters/json_test.go b/process/elasticproc/converters/json_test.go similarity index 100% rename from converters/json_test.go rename to process/elasticproc/converters/json_test.go diff --git a/converters/tags.go b/process/elasticproc/converters/tags.go similarity index 100% rename from converters/tags.go rename to process/elasticproc/converters/tags.go diff --git a/converters/tags_test.go b/process/elasticproc/converters/tags_test.go similarity index 100% rename from converters/tags_test.go rename to process/elasticproc/converters/tags_test.go diff --git a/converters/tokenIdentifier.go b/process/elasticproc/converters/tokenIdentifier.go similarity index 100% rename from converters/tokenIdentifier.go rename to process/elasticproc/converters/tokenIdentifier.go diff --git a/converters/tokenIdentifier_test.go b/process/elasticproc/converters/tokenIdentifier_test.go similarity index 100% rename from converters/tokenIdentifier_test.go rename to process/elasticproc/converters/tokenIdentifier_test.go diff --git a/converters/tokenMetaData.go b/process/elasticproc/converters/tokenMetaData.go similarity index 100% rename from converters/tokenMetaData.go rename to process/elasticproc/converters/tokenMetaData.go diff --git a/converters/tokenMetaData_test.go b/process/elasticproc/converters/tokenMetaData_test.go similarity index 100% rename from converters/tokenMetaData_test.go rename to process/elasticproc/converters/tokenMetaData_test.go diff --git a/process/elasticProcessor.go b/process/elasticproc/elasticProcessor.go similarity index 98% rename from process/elasticProcessor.go rename to process/elasticproc/elasticProcessor.go index 89731e3b..c3f56de6 100644 --- a/process/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -1,16 +1,16 @@ -package process +package elasticproc import ( "bytes" "encoding/hex" "fmt" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/process/collections" - "github.com/ElrondNetwork/elastic-indexer-go/process/tags" - "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/collections" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tags" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" diff --git a/process/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go similarity index 90% rename from process/elasticProcessor_test.go rename to process/elasticproc/elasticProcessor_test.go index 36add250..45167359 100644 --- a/process/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -1,4 +1,4 @@ -package process +package elasticproc import ( "bytes" @@ -12,19 +12,19 @@ import ( "strings" "testing" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/process/accounts" - "github.com/ElrondNetwork/elastic-indexer-go/process/block" - "github.com/ElrondNetwork/elastic-indexer-go/process/logsevents" - "github.com/ElrondNetwork/elastic-indexer-go/process/miniblocks" - "github.com/ElrondNetwork/elastic-indexer-go/process/operations" - "github.com/ElrondNetwork/elastic-indexer-go/process/statistics" - "github.com/ElrondNetwork/elastic-indexer-go/process/tags" - "github.com/ElrondNetwork/elastic-indexer-go/process/transactions" - "github.com/ElrondNetwork/elastic-indexer-go/process/validators" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/accounts" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/block" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/logsevents" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/miniblocks" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/operations" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/statistics" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tags" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/transactions" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/validators" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -51,7 +51,7 @@ func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, argume func createMockElasticProcessorArgs() *ArgElasticProcessor { balanceConverter, _ := converters.NewBalanceConverter(10) - acp, _ := accounts.NewAccountsProcessor(&mock.PubkeyConverterMock{}, balanceConverter,0) + acp, _ := accounts.NewAccountsProcessor(&mock.PubkeyConverterMock{}, balanceConverter, 0) bp, _ := block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) mp, _ := miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) vp, _ := validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32), 0) @@ -68,7 +68,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { return &ArgElasticProcessor{ DBClient: &mock.DatabaseWriterStub{}, EnabledIndexes: map[string]struct{}{ - elasticIndexer.BlockIndex: {}, elasticIndexer.TransactionsIndex: {}, elasticIndexer.MiniblocksIndex: {}, elasticIndexer.ValidatorsIndex: {}, elasticIndexer.RoundsIndex: {}, elasticIndexer.AccountsIndex: {}, elasticIndexer.RatingIndex: {}, elasticIndexer.AccountsHistoryIndex: {}, + dataindexer.BlockIndex: {}, dataindexer.TransactionsIndex: {}, dataindexer.MiniblocksIndex: {}, dataindexer.ValidatorsIndex: {}, dataindexer.RoundsIndex: {}, dataindexer.AccountsIndex: {}, dataindexer.RatingIndex: {}, dataindexer.AccountsHistoryIndex: {}, }, ValidatorsProc: vp, StatisticsProc: statistics.NewStatisticsProcessor(), @@ -154,7 +154,7 @@ func TestNewElasticProcessor(t *testing.T) { args: func() *ArgElasticProcessor { return nil }, - exErr: elasticIndexer.ErrNilElasticProcessorArguments, + exErr: dataindexer.ErrNilElasticProcessorArguments, }, { name: "NilEnabledIndexesMap", @@ -163,7 +163,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.EnabledIndexes = nil return arguments }, - exErr: elasticIndexer.ErrNilEnabledIndexesMap, + exErr: dataindexer.ErrNilEnabledIndexesMap, }, { name: "NilDatabaseClient", @@ -172,7 +172,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.DBClient = nil return arguments }, - exErr: elasticIndexer.ErrNilDatabaseClient, + exErr: dataindexer.ErrNilDatabaseClient, }, { name: "NilStatisticProc", @@ -181,7 +181,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.StatisticsProc = nil return arguments }, - exErr: elasticIndexer.ErrNilStatisticHandler, + exErr: dataindexer.ErrNilStatisticHandler, }, { name: "NilBlockProc", @@ -190,7 +190,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.BlockProc = nil return arguments }, - exErr: elasticIndexer.ErrNilBlockHandler, + exErr: dataindexer.ErrNilBlockHandler, }, { name: "NilAccountsProc", @@ -199,7 +199,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.AccountsProc = nil return arguments }, - exErr: elasticIndexer.ErrNilAccountsHandler, + exErr: dataindexer.ErrNilAccountsHandler, }, { name: "NilMiniblocksProc", @@ -208,7 +208,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.MiniblocksProc = nil return arguments }, - exErr: elasticIndexer.ErrNilMiniblocksHandler, + exErr: dataindexer.ErrNilMiniblocksHandler, }, { name: "NilValidatorsProc", @@ -217,7 +217,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.ValidatorsProc = nil return arguments }, - exErr: elasticIndexer.ErrNilValidatorsHandler, + exErr: dataindexer.ErrNilValidatorsHandler, }, { name: "NilTxsProc", @@ -226,7 +226,7 @@ func TestNewElasticProcessor(t *testing.T) { arguments.TransactionsProc = nil return arguments }, - exErr: elasticIndexer.ErrNilTransactionsHandler, + exErr: dataindexer.ErrNilTransactionsHandler, }, { name: "InitError", @@ -387,7 +387,7 @@ func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { dbWriter := &mock.DatabaseWriterStub{ DoRequestCalled: func(req *esapi.IndexRequest) error { - require.Equal(t, elasticIndexer.BlockIndex, req.Index) + require.Equal(t, dataindexer.BlockIndex, req.Index) var bl data.Block blockBytes, _ := ioutil.ReadAll(req.Body) @@ -564,7 +564,7 @@ func TestElasticProcessor_RemoveTransactions(t *testing.T) { dbWriter := &mock.DatabaseWriterStub{ DoQueryRemoveCalled: func(index string, body *bytes.Buffer) error { bodyStr := body.String() - require.Contains(t, []string{elasticIndexer.TransactionsIndex, elasticIndexer.OperationsIndex}, index) + require.Contains(t, []string{dataindexer.TransactionsIndex, dataindexer.OperationsIndex}, index) require.True(t, strings.Contains(bodyStr, expectedHashes[0])) require.True(t, strings.Contains(bodyStr, expectedHashes[1])) called = true @@ -617,12 +617,12 @@ func TestElasticProcessor_IndexEpochInfoData(t *testing.T) { elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.selfShardID = core.MetachainShardId - elasticSearchProc.enabledIndexes[elasticIndexer.EpochInfoIndex] = struct{}{} + elasticSearchProc.enabledIndexes[dataindexer.EpochInfoIndex] = struct{}{} buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) shardHeader := &dataBlock.Header{} err := elasticSearchProc.indexEpochInfoData(shardHeader, buffSlice) - require.True(t, errors.Is(err, elasticIndexer.ErrHeaderTypeAssertion)) + require.True(t, errors.Is(err, dataindexer.ErrHeaderTypeAssertion)) body := &dataBlock.Body{} metaHeader := &dataBlock.MetaBlock{} @@ -656,7 +656,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) } elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) - elasticSearchProc.enabledIndexes[elasticIndexer.ScResultsIndex] = struct{}{} + elasticSearchProc.enabledIndexes[dataindexer.ScResultsIndex] = struct{}{} err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &outport.Pool{}, nil) require.Nil(t, err) @@ -681,8 +681,8 @@ func TestElasticProcessor_IndexAlteredAccounts(t *testing.T) { }, } elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) - elasticSearchProc.enabledIndexes[elasticIndexer.AccountsESDTIndex] = struct{}{} - elasticSearchProc.enabledIndexes[elasticIndexer.AccountsESDTHistoryIndex] = struct{}{} + elasticSearchProc.enabledIndexes[dataindexer.AccountsESDTIndex] = struct{}{} + elasticSearchProc.enabledIndexes[dataindexer.AccountsESDTHistoryIndex] = struct{}{} buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) alteredAccounts := data.NewAlteredAccounts() diff --git a/process/interface.go b/process/elasticproc/interface.go similarity index 98% rename from process/interface.go rename to process/elasticproc/interface.go index d59143df..f0b408e7 100644 --- a/process/interface.go +++ b/process/elasticproc/interface.go @@ -1,10 +1,10 @@ -package process +package elasticproc import ( "bytes" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/process/logsevents/delegatorsProcessor.go b/process/elasticproc/logsevents/delegatorsProcessor.go similarity index 97% rename from process/logsevents/delegatorsProcessor.go rename to process/elasticproc/logsevents/delegatorsProcessor.go index 320aba6e..f8618042 100644 --- a/process/logsevents/delegatorsProcessor.go +++ b/process/elasticproc/logsevents/delegatorsProcessor.go @@ -4,8 +4,8 @@ import ( "math/big" "strconv" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" ) diff --git a/process/logsevents/delegatorsProcessor_test.go b/process/elasticproc/logsevents/delegatorsProcessor_test.go similarity index 97% rename from process/logsevents/delegatorsProcessor_test.go rename to process/elasticproc/logsevents/delegatorsProcessor_test.go index c8bb210c..4d50571a 100644 --- a/process/logsevents/delegatorsProcessor_test.go +++ b/process/elasticproc/logsevents/delegatorsProcessor_test.go @@ -5,9 +5,9 @@ import ( "strconv" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/process/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go similarity index 100% rename from process/logsevents/esdtIssueProcessor.go rename to process/elasticproc/logsevents/esdtIssueProcessor.go diff --git a/process/logsevents/esdtIssueProcessor_test.go b/process/elasticproc/logsevents/esdtIssueProcessor_test.go similarity index 100% rename from process/logsevents/esdtIssueProcessor_test.go rename to process/elasticproc/logsevents/esdtIssueProcessor_test.go diff --git a/process/logsevents/esdtPropertiesProcessor.go b/process/elasticproc/logsevents/esdtPropertiesProcessor.go similarity index 100% rename from process/logsevents/esdtPropertiesProcessor.go rename to process/elasticproc/logsevents/esdtPropertiesProcessor.go diff --git a/process/logsevents/esdtPropertiesProcessor_test.go b/process/elasticproc/logsevents/esdtPropertiesProcessor_test.go similarity index 97% rename from process/logsevents/esdtPropertiesProcessor_test.go rename to process/elasticproc/logsevents/esdtPropertiesProcessor_test.go index d115ddab..785b4776 100644 --- a/process/logsevents/esdtPropertiesProcessor_test.go +++ b/process/elasticproc/logsevents/esdtPropertiesProcessor_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" diff --git a/process/logsevents/fungibleESDTProcessor.go b/process/elasticproc/logsevents/fungibleESDTProcessor.go similarity index 93% rename from process/logsevents/fungibleESDTProcessor.go rename to process/elasticproc/logsevents/fungibleESDTProcessor.go index c319acc1..bbc6733b 100644 --- a/process/logsevents/fungibleESDTProcessor.go +++ b/process/elasticproc/logsevents/fungibleESDTProcessor.go @@ -3,8 +3,8 @@ package logsevents import ( "math/big" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" ) @@ -15,11 +15,11 @@ const ( type fungibleESDTProcessor struct { pubKeyConverter core.PubkeyConverter - shardCoordinator indexer.ShardCoordinator + shardCoordinator dataindexer.ShardCoordinator fungibleOperationsIdentifiers map[string]struct{} } -func newFungibleESDTProcessor(pubKeyConverter core.PubkeyConverter, shardCoordinator indexer.ShardCoordinator) *fungibleESDTProcessor { +func newFungibleESDTProcessor(pubKeyConverter core.PubkeyConverter, shardCoordinator dataindexer.ShardCoordinator) *fungibleESDTProcessor { return &fungibleESDTProcessor{ pubKeyConverter: pubKeyConverter, shardCoordinator: shardCoordinator, diff --git a/process/logsevents/fungibleESDTProcessor_test.go b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go similarity index 100% rename from process/logsevents/fungibleESDTProcessor_test.go rename to process/elasticproc/logsevents/fungibleESDTProcessor_test.go diff --git a/process/logsevents/informativeLogsProcessor.go b/process/elasticproc/logsevents/informativeLogsProcessor.go similarity index 100% rename from process/logsevents/informativeLogsProcessor.go rename to process/elasticproc/logsevents/informativeLogsProcessor.go diff --git a/process/logsevents/informativeLogsProcessor_test.go b/process/elasticproc/logsevents/informativeLogsProcessor_test.go similarity index 100% rename from process/logsevents/informativeLogsProcessor_test.go rename to process/elasticproc/logsevents/informativeLogsProcessor_test.go diff --git a/process/logsevents/interface.go b/process/elasticproc/logsevents/interface.go similarity index 92% rename from process/logsevents/interface.go rename to process/elasticproc/logsevents/interface.go index 42b62797..62e9b4ad 100644 --- a/process/logsevents/interface.go +++ b/process/elasticproc/logsevents/interface.go @@ -2,7 +2,7 @@ package logsevents import ( "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" coreData "github.com/ElrondNetwork/elrond-go-core/data" ) diff --git a/process/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go similarity index 94% rename from process/logsevents/logsAndEventsProcessor.go rename to process/elasticproc/logsevents/logsAndEventsProcessor.go index 7fc352ec..86fc47ae 100644 --- a/process/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -4,8 +4,8 @@ import ( "encoding/hex" "time" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -15,10 +15,10 @@ import ( // ArgsLogsAndEventsProcessor holds all dependencies required to create new instances of logsAndEventsProcessor type ArgsLogsAndEventsProcessor struct { - ShardCoordinator elasticIndexer.ShardCoordinator + ShardCoordinator dataindexer.ShardCoordinator PubKeyConverter core.PubkeyConverter Marshalizer marshal.Marshalizer - BalanceConverter elasticIndexer.BalanceConverter + BalanceConverter dataindexer.BalanceConverter Hasher hashing.Hasher } @@ -48,19 +48,19 @@ func NewLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) (*logsAndEvents func checkArgsLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) error { if check.IfNil(args.ShardCoordinator) { - return elasticIndexer.ErrNilShardCoordinator + return dataindexer.ErrNilShardCoordinator } if check.IfNil(args.PubKeyConverter) { - return elasticIndexer.ErrNilPubkeyConverter + return dataindexer.ErrNilPubkeyConverter } if check.IfNil(args.Marshalizer) { - return elasticIndexer.ErrNilMarshalizer + return dataindexer.ErrNilMarshalizer } if check.IfNil(args.BalanceConverter) { - return elasticIndexer.ErrNilBalanceConverter + return dataindexer.ErrNilBalanceConverter } if check.IfNil(args.Hasher) { - return elasticIndexer.ErrNilHasher + return dataindexer.ErrNilHasher } return nil diff --git a/process/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go similarity index 98% rename from process/logsevents/logsAndEventsProcessor_test.go rename to process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 9e8691d8..daa2f3e6 100644 --- a/process/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -5,10 +5,10 @@ import ( "testing" "time" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/transaction" diff --git a/process/logsevents/logsData.go b/process/elasticproc/logsevents/logsData.go similarity index 89% rename from process/logsevents/logsData.go rename to process/elasticproc/logsevents/logsData.go index 9d0c70e4..11aaa163 100644 --- a/process/logsevents/logsData.go +++ b/process/elasticproc/logsevents/logsData.go @@ -1,9 +1,9 @@ package logsevents import ( - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" ) type logsData struct { diff --git a/process/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go similarity index 91% rename from process/logsevents/nftsProcessor.go rename to process/elasticproc/logsevents/nftsProcessor.go index e0b512b9..b91fe527 100644 --- a/process/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -4,9 +4,9 @@ import ( "math/big" "time" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/esdt" @@ -70,7 +70,7 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv } token := string(topics[0]) - identifier := converters.ComputeTokenIdentifier(token, nonceBig.Uint64()) + identifier := converters2.ComputeTokenIdentifier(token, nonceBig.Uint64()) valueBig := big.NewInt(0).SetBytes(topics[2]) if !np.shouldAddReceiverData(args) { @@ -138,7 +138,7 @@ func (np *nftsProcessor) processNFTEventOnSender( if eventIdentifier == core.BuiltInFunctionESDTNFTBurn || eventIdentifier == core.BuiltInFunctionESDTWipe { tokensSupply.Add(&data.TokenInfo{ Token: token, - Identifier: converters.ComputeTokenIdentifier(token, nonceBig.Uint64()), + Identifier: converters2.ComputeTokenIdentifier(token, nonceBig.Uint64()), Timestamp: time.Duration(timestamp), Nonce: nonceBig.Uint64(), }) @@ -166,10 +166,10 @@ func (np *nftsProcessor) processNFTEventOnSender( return } - tokenMetaData := converters.PrepareTokenMetaData(np.pubKeyConverter, esdtToken) + tokenMetaData := converters2.PrepareTokenMetaData(np.pubKeyConverter, esdtToken) tokensCreateInfo.Add(&data.TokenInfo{ Token: token, - Identifier: converters.ComputeTokenIdentifier(token, nonceBig.Uint64()), + Identifier: converters2.ComputeTokenIdentifier(token, nonceBig.Uint64()), Timestamp: time.Duration(timestamp), Data: tokenMetaData, Nonce: nonceBig.Uint64(), diff --git a/process/logsevents/nftsProcessor_test.go b/process/elasticproc/logsevents/nftsProcessor_test.go similarity index 100% rename from process/logsevents/nftsProcessor_test.go rename to process/elasticproc/logsevents/nftsProcessor_test.go diff --git a/process/logsevents/nftsPropertiesProcessor.go b/process/elasticproc/logsevents/nftsPropertiesProcessor.go similarity index 96% rename from process/logsevents/nftsPropertiesProcessor.go rename to process/elasticproc/logsevents/nftsPropertiesProcessor.go index 9ada7afb..0de43341 100644 --- a/process/logsevents/nftsPropertiesProcessor.go +++ b/process/elasticproc/logsevents/nftsPropertiesProcessor.go @@ -3,8 +3,8 @@ package logsevents import ( "math/big" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" ) diff --git a/process/logsevents/nftsPropertiesProcessor_test.go b/process/elasticproc/logsevents/nftsPropertiesProcessor_test.go similarity index 100% rename from process/logsevents/nftsPropertiesProcessor_test.go rename to process/elasticproc/logsevents/nftsPropertiesProcessor_test.go diff --git a/process/logsevents/scDeploys.go b/process/elasticproc/logsevents/scDeploys.go similarity index 100% rename from process/logsevents/scDeploys.go rename to process/elasticproc/logsevents/scDeploys.go diff --git a/process/logsevents/scDeploys_test.go b/process/elasticproc/logsevents/scDeploys_test.go similarity index 100% rename from process/logsevents/scDeploys_test.go rename to process/elasticproc/logsevents/scDeploys_test.go diff --git a/process/logsevents/serialize.go b/process/elasticproc/logsevents/serialize.go similarity index 84% rename from process/logsevents/serialize.go rename to process/elasticproc/logsevents/serialize.go index e7f4107f..1c58db1f 100644 --- a/process/logsevents/serialize.go +++ b/process/elasticproc/logsevents/serialize.go @@ -5,16 +5,16 @@ import ( "encoding/json" "fmt" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/process/tokeninfo" + converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" "github.com/ElrondNetwork/elrond-go-core/core" ) // SerializeLogs will serialize the provided logs in a way that Elasticsearch expects a bulk request func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data.BufferSlice, index string) error { for _, lg := range logs { - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(lg.ID), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(lg.ID), "\n")) serializedData, errMarshal := json.Marshal(lg) if errMarshal != nil { return errMarshal @@ -38,7 +38,7 @@ func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data. `"lang": "painless",`+ `"params": { "log": %s }},`+ `"upsert": {}}`, - converters.FormatPainlessSource(codeToExecute), serializedData, + converters2.FormatPainlessSource(codeToExecute), serializedData, ) err := buffSlice.PutData(meta, []byte(serializedDataStr)) @@ -53,7 +53,7 @@ func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data. // SerializeSCDeploys will serialize the provided smart contract deploys in a way that Elasticsearch expects a bulk request func (*logsAndEventsProcessor) SerializeSCDeploys(deploys map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice, index string) error { for scAddr, deployInfo := range deploys { - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(scAddr), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(scAddr), "\n")) serializedData, err := serializeDeploy(deployInfo) if err != nil { @@ -98,7 +98,7 @@ func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) { `"lang": "painless",`+ `"params": {"elem": %s}},`+ `"upsert": %s}`, - converters.FormatPainlessSource(codeToExecute), string(upgradeSerialized), string(serializedData)) + converters2.FormatPainlessSource(codeToExecute), string(upgradeSerialized), string(serializedData)) return []byte(serializedDataStr), nil } @@ -117,7 +117,7 @@ func (*logsAndEventsProcessor) SerializeTokens(tokens []*data.TokenInfo, updateN } } - return converters.PrepareNFTUpdateData(buffSlice, updateNFTData, false, index) + return converters2.PrepareNFTUpdateData(buffSlice, updateNFTData, false, index) } func serializeToken(tokenData *data.TokenInfo, index string) ([]byte, []byte, error) { @@ -125,7 +125,7 @@ func serializeToken(tokenData *data.TokenInfo, index string) ([]byte, []byte, er return serializeTokenTransferOwnership(tokenData, index) } - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tokenData.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tokenData.Token), "\n")) serializedTokenData, err := json.Marshal(tokenData) if err != nil { return nil, nil, err @@ -143,13 +143,13 @@ func serializeToken(tokenData *data.TokenInfo, index string) ([]byte, []byte, er `"lang": "painless",`+ `"params": {"token": %s}},`+ `"upsert": %s}`, - converters.FormatPainlessSource(codeToExecute), string(serializedTokenData), string(serializedTokenData)) + converters2.FormatPainlessSource(codeToExecute), string(serializedTokenData), string(serializedTokenData)) return meta, []byte(serializedDataStr), nil } func serializeTokenTransferOwnership(tokenData *data.TokenInfo, index string) ([]byte, []byte, error) { - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tokenData.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tokenData.Token), "\n")) tokenDataSerialized, err := json.Marshal(tokenData) if err != nil { return nil, nil, err @@ -178,7 +178,7 @@ func serializeTokenTransferOwnership(tokenData *data.TokenInfo, index string) ([ `"lang": "painless",`+ `"params": {"elem": %s, "owner": "%s"}},`+ `"upsert": %s}`, - converters.FormatPainlessSource(codeToExecute), string(ownerDataSerialized), converters.JsonEscape(tokenData.CurrentOwner), string(tokenDataSerialized)) + converters2.FormatPainlessSource(codeToExecute), string(ownerDataSerialized), converters2.JsonEscape(tokenData.CurrentOwner), string(tokenDataSerialized)) return meta, []byte(serializedDataStr), nil } @@ -203,11 +203,11 @@ func (lep *logsAndEventsProcessor) SerializeDelegators(delegators map[string]*da func (lep *logsAndEventsProcessor) prepareSerializedDelegator(delegator *data.Delegator, index string) ([]byte, []byte, error) { id := lep.computeDelegatorID(delegator) if delegator.ShouldDelete { - meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(id), "\n")) + meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(id), "\n")) return meta, nil, nil } - meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(id), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(id), "\n")) serializedData, errMarshal := json.Marshal(delegator) if errMarshal != nil { return nil, nil, errMarshal @@ -231,7 +231,7 @@ func (lep *logsAndEventsProcessor) SerializeSupplyData(tokensSupply data.TokensH continue } - meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(supplyData.Identifier), "\n")) + meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(supplyData.Identifier), "\n")) err := buffSlice.PutData(meta, nil) if err != nil { return err @@ -267,7 +267,7 @@ func (lep *logsAndEventsProcessor) SerializeRolesData( } func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role string, index string) error { - meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(rd.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(rd.Token), "\n")) var serializedDataStr string if rd.Set { codeToExecute := ` @@ -291,11 +291,11 @@ func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role `"lang": "painless",`+ `"params": { "role": "%s", "address": "%s"}},`+ `"upsert": { "roles": {"%s": ["%s"]}}}`, - converters.FormatPainlessSource(codeToExecute), - converters.JsonEscape(role), - converters.JsonEscape(rd.Address), - converters.JsonEscape(role), - converters.JsonEscape(rd.Address), + converters2.FormatPainlessSource(codeToExecute), + converters2.JsonEscape(role), + converters2.JsonEscape(rd.Address), + converters2.JsonEscape(role), + converters2.JsonEscape(rd.Address), ) } else { codeToExecute := ` @@ -310,9 +310,9 @@ func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role `"lang": "painless",`+ `"params": { "role": "%s", "address": "%s" }},`+ `"upsert": {} }`, - converters.FormatPainlessSource(codeToExecute), - converters.JsonEscape(role), - converters.JsonEscape(rd.Address), + converters2.FormatPainlessSource(codeToExecute), + converters2.JsonEscape(role), + converters2.JsonEscape(rd.Address), ) } @@ -320,7 +320,7 @@ func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role } func serializePropertiesData(buffSlice *data.BufferSlice, index string, tokenProp *tokeninfo.PropertiesData) error { - meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tokenProp.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tokenProp.Token), "\n")) propertiesBytes, err := json.Marshal(tokenProp.Properties) if err != nil { @@ -340,7 +340,7 @@ func serializePropertiesData(buffSlice *data.BufferSlice, index string, tokenPro `"lang": "painless",`+ `"params": { "properties": %s}},`+ `"upsert": {}}}`, - converters.FormatPainlessSource(codeToExecute), propertiesBytes) + converters2.FormatPainlessSource(codeToExecute), propertiesBytes) return buffSlice.PutData(meta, []byte(serializedDataStr)) } diff --git a/process/logsevents/serialize_test.go b/process/elasticproc/logsevents/serialize_test.go similarity index 100% rename from process/logsevents/serialize_test.go rename to process/elasticproc/logsevents/serialize_test.go diff --git a/process/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go similarity index 97% rename from process/miniblocks/miniblocksProcessor.go rename to process/elasticproc/miniblocks/miniblocksProcessor.go index 07250636..c5ebb242 100644 --- a/process/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -4,8 +4,8 @@ import ( "encoding/hex" "time" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -32,10 +32,10 @@ func NewMiniblocksProcessor( isImportDBMode bool, ) (*miniblocksProcessor, error) { if check.IfNil(marshalier) { - return nil, indexer.ErrNilMarshalizer + return nil, dataindexer.ErrNilMarshalizer } if check.IfNil(hasher) { - return nil, indexer.ErrNilHasher + return nil, dataindexer.ErrNilHasher } return &miniblocksProcessor{ diff --git a/process/miniblocks/miniblocksProcessor_test.go b/process/elasticproc/miniblocks/miniblocksProcessor_test.go similarity index 96% rename from process/miniblocks/miniblocksProcessor_test.go rename to process/elasticproc/miniblocks/miniblocksProcessor_test.go index f0896324..aefdb6cb 100644 --- a/process/miniblocks/miniblocksProcessor_test.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor_test.go @@ -3,8 +3,8 @@ package miniblocks import ( "testing" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/mock" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" @@ -24,14 +24,14 @@ func TestNewMiniblocksProcessor(t *testing.T) { args: func() (uint32, hashing.Hasher, marshal.Marshalizer, bool) { return 0, nil, &mock.MarshalizerMock{}, false }, - exErr: indexer.ErrNilHasher, + exErr: dataindexer.ErrNilHasher, }, { name: "NilMarshalizer", args: func() (uint32, hashing.Hasher, marshal.Marshalizer, bool) { return 0, &mock.HasherMock{}, nil, false }, - exErr: indexer.ErrNilMarshalizer, + exErr: dataindexer.ErrNilMarshalizer, }, } diff --git a/process/miniblocks/serialize.go b/process/elasticproc/miniblocks/serialize.go similarity index 96% rename from process/miniblocks/serialize.go rename to process/elasticproc/miniblocks/serialize.go index 185fdbe7..995c2938 100644 --- a/process/miniblocks/serialize.go +++ b/process/elasticproc/miniblocks/serialize.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/data/block" ) diff --git a/process/miniblocks/serialize_test.go b/process/elasticproc/miniblocks/serialize_test.go similarity index 100% rename from process/miniblocks/serialize_test.go rename to process/elasticproc/miniblocks/serialize_test.go diff --git a/process/operations/operationsProcessor.go b/process/elasticproc/operations/operationsProcessor.go similarity index 92% rename from process/operations/operationsProcessor.go rename to process/elasticproc/operations/operationsProcessor.go index c35164a9..5d58382c 100644 --- a/process/operations/operationsProcessor.go +++ b/process/elasticproc/operations/operationsProcessor.go @@ -4,8 +4,8 @@ import ( "encoding/hex" "strings" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data/transaction" @@ -14,13 +14,13 @@ import ( type operationsProcessor struct { importDBMode bool - shardCoordinator indexer.ShardCoordinator + shardCoordinator dataindexer.ShardCoordinator } // NewOperationsProcessor will create a new instance of operationsProcessor -func NewOperationsProcessor(importDBMode bool, shardCoordinator indexer.ShardCoordinator) (*operationsProcessor, error) { +func NewOperationsProcessor(importDBMode bool, shardCoordinator dataindexer.ShardCoordinator) (*operationsProcessor, error) { if check.IfNil(shardCoordinator) { - return nil, indexer.ErrNilShardCoordinator + return nil, dataindexer.ErrNilShardCoordinator } return &operationsProcessor{ diff --git a/process/operations/operationsProcessor_test.go b/process/elasticproc/operations/operationsProcessor_test.go similarity index 96% rename from process/operations/operationsProcessor_test.go rename to process/elasticproc/operations/operationsProcessor_test.go index cfb0b720..034d5e08 100644 --- a/process/operations/operationsProcessor_test.go +++ b/process/elasticproc/operations/operationsProcessor_test.go @@ -3,9 +3,9 @@ package operations import ( "testing" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) diff --git a/process/operations/serialize.go b/process/elasticproc/operations/serialize.go similarity index 94% rename from process/operations/serialize.go rename to process/elasticproc/operations/serialize.go index 07fea407..5f3e0bea 100644 --- a/process/operations/serialize.go +++ b/process/elasticproc/operations/serialize.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" ) // SerializeSCRs will serialize smart contract results diff --git a/process/operations/serialize_test.go b/process/elasticproc/operations/serialize_test.go similarity index 100% rename from process/operations/serialize_test.go rename to process/elasticproc/operations/serialize_test.go diff --git a/process/statistics/serialize.go b/process/elasticproc/statistics/serialize.go similarity index 100% rename from process/statistics/serialize.go rename to process/elasticproc/statistics/serialize.go diff --git a/process/statistics/serialize_test.go b/process/elasticproc/statistics/serialize_test.go similarity index 100% rename from process/statistics/serialize_test.go rename to process/elasticproc/statistics/serialize_test.go diff --git a/process/tags/serialize.go b/process/elasticproc/tags/serialize.go similarity index 75% rename from process/tags/serialize.go rename to process/elasticproc/tags/serialize.go index 503a005a..a5707ac1 100644 --- a/process/tags/serialize.go +++ b/process/elasticproc/tags/serialize.go @@ -4,8 +4,8 @@ import ( "encoding/base64" "fmt" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" ) // Serialize will serialize tagsCount in a way that Elasticsearch expects a bulk request @@ -16,14 +16,14 @@ func (tc *tagsCount) Serialize(buffSlice *data.BufferSlice, index string) error } base64Tag := base64.StdEncoding.EncodeToString([]byte(tag)) - meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(base64Tag), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(base64Tag), "\n")) codeToExecute := ` ctx._source.count += params.count; ctx._source.tag = params.tag ` serializedDataStr := fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"count": %d, "tag": "%s"}},"upsert": {"count": %d, "tag":"%s"}}`, - converters.FormatPainlessSource(codeToExecute), count, converters.JsonEscape(tag), count, converters.JsonEscape(tag), + converters2.FormatPainlessSource(codeToExecute), count, converters2.JsonEscape(tag), count, converters2.JsonEscape(tag), ) err := buffSlice.PutData(meta, []byte(serializedDataStr)) diff --git a/process/tags/serialize_test.go b/process/elasticproc/tags/serialize_test.go similarity index 100% rename from process/tags/serialize_test.go rename to process/elasticproc/tags/serialize_test.go diff --git a/process/tags/tagsCount.go b/process/elasticproc/tags/tagsCount.go similarity index 100% rename from process/tags/tagsCount.go rename to process/elasticproc/tags/tagsCount.go diff --git a/process/tags/tagsCount_test.go b/process/elasticproc/tags/tagsCount_test.go similarity index 100% rename from process/tags/tagsCount_test.go rename to process/elasticproc/tags/tagsCount_test.go diff --git a/process/templatesAndPolicies/interface.go b/process/elasticproc/templatesAndPolicies/interface.go similarity index 100% rename from process/templatesAndPolicies/interface.go rename to process/elasticproc/templatesAndPolicies/interface.go diff --git a/process/templatesAndPolicies/noKibana.go b/process/elasticproc/templatesAndPolicies/noKibana.go similarity index 96% rename from process/templatesAndPolicies/noKibana.go rename to process/elasticproc/templatesAndPolicies/noKibana.go index ce96e2a9..18fc9a99 100644 --- a/process/templatesAndPolicies/noKibana.go +++ b/process/elasticproc/templatesAndPolicies/noKibana.go @@ -3,7 +3,7 @@ package templatesAndPolicies import ( "bytes" - indexer "github.com/ElrondNetwork/elastic-indexer-go" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elastic-indexer-go/templates/noKibana" ) diff --git a/process/templatesAndPolicies/noKibana_test.go b/process/elasticproc/templatesAndPolicies/noKibana_test.go similarity index 100% rename from process/templatesAndPolicies/noKibana_test.go rename to process/elasticproc/templatesAndPolicies/noKibana_test.go diff --git a/process/templatesAndPolicies/templatesAndPoliciesFactory.go b/process/elasticproc/templatesAndPolicies/templatesAndPoliciesFactory.go similarity index 100% rename from process/templatesAndPolicies/templatesAndPoliciesFactory.go rename to process/elasticproc/templatesAndPolicies/templatesAndPoliciesFactory.go diff --git a/process/templatesAndPolicies/templatesAndPoliciesFactory_test.go b/process/elasticproc/templatesAndPolicies/templatesAndPoliciesFactory_test.go similarity index 100% rename from process/templatesAndPolicies/templatesAndPoliciesFactory_test.go rename to process/elasticproc/templatesAndPolicies/templatesAndPoliciesFactory_test.go diff --git a/process/templatesAndPolicies/withKibana.go b/process/elasticproc/templatesAndPolicies/withKibana.go similarity index 97% rename from process/templatesAndPolicies/withKibana.go rename to process/elasticproc/templatesAndPolicies/withKibana.go index 1976c1bc..d4ff8dde 100644 --- a/process/templatesAndPolicies/withKibana.go +++ b/process/elasticproc/templatesAndPolicies/withKibana.go @@ -3,7 +3,7 @@ package templatesAndPolicies import ( "bytes" - indexer "github.com/ElrondNetwork/elastic-indexer-go" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elastic-indexer-go/templates/withKibana" ) diff --git a/process/templatesAndPolicies/withKibana_test.go b/process/elasticproc/templatesAndPolicies/withKibana_test.go similarity index 100% rename from process/templatesAndPolicies/withKibana_test.go rename to process/elasticproc/templatesAndPolicies/withKibana_test.go diff --git a/process/tokeninfo/tokeninfo.go b/process/elasticproc/tokeninfo/tokeninfo.go similarity index 100% rename from process/tokeninfo/tokeninfo.go rename to process/elasticproc/tokeninfo/tokeninfo.go diff --git a/process/tokeninfo/tokeninfo_test.go b/process/elasticproc/tokeninfo/tokeninfo_test.go similarity index 100% rename from process/tokeninfo/tokeninfo_test.go rename to process/elasticproc/tokeninfo/tokeninfo_test.go diff --git a/process/transactions/checkers.go b/process/elasticproc/transactions/checkers.go similarity index 97% rename from process/transactions/checkers.go rename to process/elasticproc/transactions/checkers.go index 0f9587c2..5c4074a5 100644 --- a/process/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -6,8 +6,8 @@ import ( "math/big" "strings" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" diff --git a/process/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go similarity index 98% rename from process/transactions/checkers_test.go rename to process/elasticproc/transactions/checkers_test.go index 051e12c5..8507b56e 100644 --- a/process/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -5,9 +5,9 @@ import ( "math/big" "testing" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/process/transactions/interface.go b/process/elasticproc/transactions/interface.go similarity index 100% rename from process/transactions/interface.go rename to process/elasticproc/transactions/interface.go diff --git a/process/transactions/scrsDataToTransactions.go b/process/elasticproc/transactions/scrsDataToTransactions.go similarity index 100% rename from process/transactions/scrsDataToTransactions.go rename to process/elasticproc/transactions/scrsDataToTransactions.go diff --git a/process/transactions/scrsDataToTransactions_test.go b/process/elasticproc/transactions/scrsDataToTransactions_test.go similarity index 100% rename from process/transactions/scrsDataToTransactions_test.go rename to process/elasticproc/transactions/scrsDataToTransactions_test.go diff --git a/process/transactions/serialize.go b/process/elasticproc/transactions/serialize.go similarity index 88% rename from process/transactions/serialize.go rename to process/elasticproc/transactions/serialize.go index 49d48ce4..c5770327 100644 --- a/process/transactions/serialize.go +++ b/process/elasticproc/transactions/serialize.go @@ -5,15 +5,15 @@ import ( "fmt" "strings" - "github.com/ElrondNetwork/elastic-indexer-go/converters" "github.com/ElrondNetwork/elastic-indexer-go/data" + converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" ) // SerializeScResults will serialize the provided smart contract results in a way that ElasticSearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeScResults(scResults []*data.ScResult, buffSlice *data.BufferSlice, index string) error { for _, sc := range scResults { - meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(sc.Hash), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(sc.Hash), "\n")) serializedData, errPrepareSc := json.Marshal(sc) if errPrepareSc != nil { return errPrepareSc @@ -31,7 +31,7 @@ func (tdp *txsDatabaseProcessor) SerializeScResults(scResults []*data.ScResult, // SerializeReceipts will serialize the receipts in a way that ElasticSearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeReceipts(receipts []*data.Receipt, buffSlice *data.BufferSlice, index string) error { for _, rec := range receipts { - meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(rec.Hash), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(rec.Hash), "\n")) serializedData, errPrepareReceipt := json.Marshal(rec) if errPrepareReceipt != nil { return errPrepareReceipt @@ -49,7 +49,7 @@ func (tdp *txsDatabaseProcessor) SerializeReceipts(receipts []*data.Receipt, buf // SerializeTransactionsFeeData will serialize transactions fee data func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[string]*data.FeeData, buffSlice *data.BufferSlice, index string) error { for txHash, feeData := range txHashRefund { - meta := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, converters.JsonEscape(txHash), "\n")) + meta := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, converters2.JsonEscape(txHash), "\n")) codeToExecute := ` if ('create' == ctx.op) { ctx.op = 'noop' @@ -64,7 +64,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s `"lang": "painless",`+ `"params": {"fee": "%s", "gasUsed": %d}},`+ `"upsert": {}}`, - converters.FormatPainlessSource(codeToExecute), feeData.Fee, feeData.GasUsed, + converters2.FormatPainlessSource(codeToExecute), feeData.Fee, feeData.GasUsed, ) err := buffSlice.PutData(meta, []byte(serializedDataStr)) @@ -119,7 +119,7 @@ func serializeTxHashStatus(buffSlice *data.BufferSlice, txHashStatus map[string] codeToExecute := ` ctx._source.status = params.status ` - serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"status": "%s"}},"upsert": %s }`, converters.FormatPainlessSource(codeToExecute), converters.JsonEscape(status), string(marshaledTx))) + serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"status": "%s"}},"upsert": %s }`, converters2.FormatPainlessSource(codeToExecute), converters2.JsonEscape(status), string(marshaledTx))) err = buffSlice.PutData(metaData, serializedData) if err != nil { return err @@ -134,7 +134,7 @@ func prepareSerializedDataForATransaction( selfShardID uint32, index string, ) ([]byte, []byte, error) { - metaData := []byte(fmt.Sprintf(`{"update":{ "_index":"%s", "_id":"%s"}}%s`, index, converters.JsonEscape(tx.Hash), "\n")) + metaData := []byte(fmt.Sprintf(`{"update":{ "_index":"%s", "_id":"%s"}}%s`, index, converters2.JsonEscape(tx.Hash), "\n")) marshaledTx, err := json.Marshal(tx) if err != nil { return nil, nil, err @@ -159,7 +159,7 @@ func prepareSerializedDataForATransaction( } // transaction is intra-shard, invalid or cross-shard destination me - meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tx.Hash), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tx.Hash), "\n")) return meta, marshaledTx, nil } @@ -175,7 +175,7 @@ func prepareNFTESDTTransferOrMultiESDTTransfer(marshaledTx []byte) ([]byte, erro } ` serializedData := []byte(fmt.Sprintf(`{"scripted_upsert": true, "script":{"source":"%s","lang": "painless","params":{"tx": %s}},"upsert":{}}`, - converters.FormatPainlessSource(codeToExecute), string(marshaledTx))) + converters2.FormatPainlessSource(codeToExecute), string(marshaledTx))) return serializedData, nil } diff --git a/process/transactions/serialize_test.go b/process/elasticproc/transactions/serialize_test.go similarity index 100% rename from process/transactions/serialize_test.go rename to process/elasticproc/transactions/serialize_test.go diff --git a/process/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go similarity index 98% rename from process/transactions/smartContractResultsProcessor.go rename to process/elasticproc/transactions/smartContractResultsProcessor.go index 420ee905..5c982284 100644 --- a/process/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -6,8 +6,8 @@ import ( "strconv" "time" - indexer "github.com/ElrondNetwork/elastic-indexer-go" indexerData "github.com/ElrondNetwork/elastic-indexer-go/data" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data" coreData "github.com/ElrondNetwork/elrond-go-core/data" diff --git a/process/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go similarity index 100% rename from process/transactions/smartContractResultsProcessor_test.go rename to process/elasticproc/transactions/smartContractResultsProcessor_test.go diff --git a/process/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go similarity index 98% rename from process/transactions/transactionDBBuilder.go rename to process/elasticproc/transactions/transactionDBBuilder.go index a4d4f47a..73c81e6a 100644 --- a/process/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -6,8 +6,8 @@ import ( "math/big" "time" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/process/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go similarity index 100% rename from process/transactions/transactionDBBuilder_test.go rename to process/elasticproc/transactions/transactionDBBuilder_test.go diff --git a/process/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go similarity index 100% rename from process/transactions/transactionsGrouper.go rename to process/elasticproc/transactions/transactionsGrouper.go diff --git a/process/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go similarity index 100% rename from process/transactions/transactionsGrouper_test.go rename to process/elasticproc/transactions/transactionsGrouper_test.go diff --git a/process/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go similarity index 99% rename from process/transactions/transactionsProcessor.go rename to process/elasticproc/transactions/transactionsProcessor.go index a0c55ed5..2517c184 100644 --- a/process/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -3,8 +3,8 @@ package transactions import ( "encoding/hex" - indexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" diff --git a/process/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go similarity index 100% rename from process/transactions/transactionsProcessor_test.go rename to process/elasticproc/transactions/transactionsProcessor_test.go diff --git a/process/updateTokenType.go b/process/elasticproc/updateTokenType.go similarity index 95% rename from process/updateTokenType.go rename to process/elasticproc/updateTokenType.go index b36ca527..fc655991 100644 --- a/process/updateTokenType.go +++ b/process/elasticproc/updateTokenType.go @@ -1,12 +1,12 @@ -package process +package elasticproc import ( "encoding/json" "fmt" "time" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" ) diff --git a/process/validators/serialize.go b/process/elasticproc/validators/serialize.go similarity index 100% rename from process/validators/serialize.go rename to process/elasticproc/validators/serialize.go diff --git a/process/validators/serialize_test.go b/process/elasticproc/validators/serialize_test.go similarity index 100% rename from process/validators/serialize_test.go rename to process/elasticproc/validators/serialize_test.go diff --git a/process/validators/validatorsProcessor.go b/process/elasticproc/validators/validatorsProcessor.go similarity index 91% rename from process/validators/validatorsProcessor.go rename to process/elasticproc/validators/validatorsProcessor.go index fb736021..99cac30c 100644 --- a/process/validators/validatorsProcessor.go +++ b/process/elasticproc/validators/validatorsProcessor.go @@ -1,8 +1,8 @@ package validators import ( - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" ) @@ -15,7 +15,7 @@ type validatorsProcessor struct { // NewValidatorsProcessor will create a new instance of validatorsProcessor func NewValidatorsProcessor(validatorPubkeyConverter core.PubkeyConverter, bulkSizeMaxSize int) (*validatorsProcessor, error) { if check.IfNil(validatorPubkeyConverter) { - return nil, indexer.ErrNilPubkeyConverter + return nil, dataindexer.ErrNilPubkeyConverter } return &validatorsProcessor{ diff --git a/process/validators/validatorsProcessor_test.go b/process/elasticproc/validators/validatorsProcessor_test.go similarity index 84% rename from process/validators/validatorsProcessor_test.go rename to process/elasticproc/validators/validatorsProcessor_test.go index 4da7b24a..cb95d100 100644 --- a/process/validators/validatorsProcessor_test.go +++ b/process/elasticproc/validators/validatorsProcessor_test.go @@ -3,9 +3,9 @@ package validators import ( "testing" - "github.com/ElrondNetwork/elastic-indexer-go" "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/stretchr/testify/require" ) @@ -14,7 +14,7 @@ func TestNewValidatorsProcessor(t *testing.T) { vp, err := NewValidatorsProcessor(nil, 0) require.Nil(t, vp) - require.Equal(t, indexer.ErrNilPubkeyConverter, err) + require.Equal(t, dataindexer.ErrNilPubkeyConverter, err) } func TestValidatorsProcessor_PrepareValidatorsPublicKeys(t *testing.T) { diff --git a/process/factory/elasticProcessorFactory.go b/process/factory/elasticProcessorFactory.go index d7cea715..62057821 100644 --- a/process/factory/elasticProcessorFactory.go +++ b/process/factory/elasticProcessorFactory.go @@ -1,18 +1,18 @@ package factory import ( - indexer "github.com/ElrondNetwork/elastic-indexer-go" - "github.com/ElrondNetwork/elastic-indexer-go/converters" - processIndexer "github.com/ElrondNetwork/elastic-indexer-go/process" - "github.com/ElrondNetwork/elastic-indexer-go/process/accounts" - blockProc "github.com/ElrondNetwork/elastic-indexer-go/process/block" - "github.com/ElrondNetwork/elastic-indexer-go/process/logsevents" - "github.com/ElrondNetwork/elastic-indexer-go/process/miniblocks" - "github.com/ElrondNetwork/elastic-indexer-go/process/operations" - "github.com/ElrondNetwork/elastic-indexer-go/process/statistics" - "github.com/ElrondNetwork/elastic-indexer-go/process/templatesAndPolicies" - "github.com/ElrondNetwork/elastic-indexer-go/process/transactions" - "github.com/ElrondNetwork/elastic-indexer-go/process/validators" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/accounts" + blockProc "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/block" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/logsevents" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/miniblocks" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/operations" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/statistics" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/templatesAndPolicies" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/transactions" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/validators" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" @@ -24,8 +24,8 @@ type ArgElasticProcessorFactory struct { Hasher hashing.Hasher AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter - DBClient processIndexer.DatabaseClientHandler - ShardCoordinator indexer.ShardCoordinator + DBClient elasticproc.DatabaseClientHandler + ShardCoordinator dataindexer.ShardCoordinator EnabledIndexes []string Denomination int BulkRequestMaxSize int @@ -34,7 +34,7 @@ type ArgElasticProcessorFactory struct { } // CreateElasticProcessor will create a new instance of ElasticProcessor -func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.ElasticProcessor, error) { +func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.ElasticProcessor, error) { templatesAndPoliciesReader := templatesAndPolicies.CreateTemplatesAndPoliciesReader(arguments.UseKibana) indexTemplates, indexPolicies, err := templatesAndPoliciesReader.GetElasticTemplatesAndPolicies() if err != nil { @@ -46,7 +46,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.Elast enabledIndexesMap[index] = struct{}{} } if len(enabledIndexesMap) == 0 { - return nil, indexer.ErrEmptyEnabledIndexes + return nil, dataindexer.ErrEmptyEnabledIndexes } balanceConverter, err := converters.NewBalanceConverter(arguments.Denomination) @@ -108,7 +108,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.Elast return nil, err } - args := &processIndexer.ArgElasticProcessor{ + args := &elasticproc.ArgElasticProcessor{ BulkRequestMaxSize: arguments.BulkRequestMaxSize, TransactionsProc: txsProc, AccountsProc: accountsProc, @@ -126,5 +126,5 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (indexer.Elast OperationsProc: operationsProc, } - return processIndexer.NewElasticProcessor(args) + return elasticproc.NewElasticProcessor(args) } From 8d2adcc78c56ff6323e9711e824a47b190b3dbde Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 1 Sep 2022 17:03:16 +0300 Subject: [PATCH 33/96] fix build --- .github/workflows/pr-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index febc9c77..5c626e17 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -29,4 +29,6 @@ jobs: dep ensure fi - name: Build - run: go build + run: | + cd cmd/elasticindexer + go build From a1f1dc9f6b5ea72b43f92aec6fd61f57a1d7972c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 2 Sep 2022 14:58:25 +0300 Subject: [PATCH 34/96] ws client and indexer --- go.mod | 3 +- go.sum | 8 +- process/wsclient/client.go | 144 +++++++++++++++++++++++++++++++++ process/wsindexer/indexer.go | 95 ++++++++++++++++++++++ process/wsindexer/interface.go | 23 ++++++ 5 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 process/wsclient/client.go create mode 100644 process/wsindexer/indexer.go create mode 100644 process/wsindexer/interface.go diff --git a/go.mod b/go.mod index 1971c3e9..02927f99 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00 + github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220831091042-a8722ecd7fba github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 + github.com/gorilla/websocket v1.5.0 github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.9 diff --git a/go.sum b/go.sum index f510b9aa..12c82167 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00 h1:pH+/JIebRGRq5PaC+qZZ7ZBr+YWDDI/N+hZtCD4onUo= -github.com/ElrondNetwork/elrond-go-core v1.1.19-0.20220829095131-68a85ab9fe00/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220831091042-a8722ecd7fba h1:mRBgGpJjj6mwdDIHxMbUqoxwAdpvK1oVUmtv78Hk9xQ= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220831091042-a8722ecd7fba/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= @@ -44,6 +44,10 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/process/wsclient/client.go b/process/wsclient/client.go new file mode 100644 index 00000000..1b68802a --- /dev/null +++ b/process/wsclient/client.go @@ -0,0 +1,144 @@ +package wsclient + +import ( + "fmt" + "io" + "net/url" + "os" + "os/signal" + "time" + + "github.com/ElrondNetwork/elrond-go-core/data/typeConverters/uint64ByteSlice" + "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver" + logger "github.com/ElrondNetwork/elrond-go-logger" + "github.com/gorilla/websocket" +) + +type Handler func(marshalledData []byte) error + +type wsConn interface { + io.Closer + ReadMessage() (messageType int, p []byte, err error) + WriteMessage(messageType int, data []byte) error +} + +var ( + log = logger.GetOrCreate("process/wsclient") +) + +type client struct { + urlReceive string + actions map[uint32]Handler +} + +func NewWebSocketClient(urlReceive string, actions map[uint32]Handler) (*client, error) { + urlReceiveData := url.URL{Scheme: "ws", Host: fmt.Sprintf(urlReceive), Path: "/operations"} + + return &client{ + actions: actions, + urlReceive: urlReceiveData.String(), + }, nil +} + +func (c *client) Start() { + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + + log.Info("connecting to", "url", c.urlReceive) + + wsConnection, _, err := websocket.DefaultDialer.Dial(c.urlReceive, nil) + if err != nil { + log.Error("dial", "error", err) + } + + defer func() { + err = wsConnection.Close() + log.LogIfError(err) + }() + + done := make(chan struct{}) + go func() { + defer close(done) + for { + var message []byte + _, message, err = wsConnection.ReadMessage() + if err != nil { + log.Error("error read message", "error", err) + return + } + + c.verifyPayloadAndSendAckIfNeeded(message, wsConnection) + } + }() + + timer := time.NewTimer(time.Second) + defer timer.Stop() + + for { + select { + case <-done: + return + case <-timer.C: + case <-interrupt: + log.Info("interrupt") + + // Cleanly close the connection by sending a close message and then + // waiting (with timeout) for the server to close the connection. + err = wsConnection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + if err != nil { + log.Error("write close", "error", err) + return + } + select { + case <-done: + case <-time.After(time.Second): + } + return + } + } +} + +func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsConn) { + uint64ByteSliceConverter := uint64ByteSlice.NewBigEndianConverter() + if len(payload) == 0 { + log.Error("empty payload") + return + } + + payloadParser, _ := websocketOutportDriver.NewWebSocketPayloadParser(uint64ByteSliceConverter) + payloadData, err := payloadParser.ExtractPayloadData(payload) + if err != nil { + log.Error("error while extracting payload data: " + err.Error()) + return + } + + log.Info("processing payload", + "counter", payloadData.Counter, + "operation type", payloadData.OperationType, + "message length", len(payloadData.Payload), + "data", payloadData.Payload, + ) + + function, ok := c.actions[payloadData.OperationType.Uint32()] + if !ok { + log.Warn("invalid operation", "operation type", payloadData.OperationType.String()) + } + + err = function(payloadData.Payload) + if err != nil { + log.Error("something went wrong", "error", err.Error()) + } + + if payloadData.WithAcknowledge { + counterBytes := uint64ByteSliceConverter.ToByteSlice(payloadData.Counter) + err = ackHandler.WriteMessage(websocket.BinaryMessage, counterBytes) + if err != nil { + log.Error("write acknowledge message", "error", err.Error()) + } + } +} + +// Stop - +func (c *client) Stop() { + // TODO implement +} diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go new file mode 100644 index 00000000..f809032d --- /dev/null +++ b/process/wsindexer/indexer.go @@ -0,0 +1,95 @@ +package wsindexer + +import ( + "github.com/ElrondNetwork/elrond-go-core/data/outport" + "github.com/ElrondNetwork/elrond-go-core/marshal" + "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" +) + +type indexer struct { + marshaller marshal.Marshalizer + di DataIndexer +} + +func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*indexer, error) { + return &indexer{ + marshaller: marshaller, + di: dataIndexer, + }, nil +} + +func (i *indexer) GetFunctionsMap() map[data.OperationType]func(d []byte) error { + return map[data.OperationType]func(d []byte) error{ + data.OperationSaveBlock: i.saveBlock, + data.OperationRevertIndexedBlock: i.revertIndexedBlock, + data.OperationSaveRoundsInfo: i.saveRounds, + data.OperationSaveValidatorsRating: i.saveValidatorsRating, + data.OperationSaveValidatorsPubKeys: i.saveValidatorsPubKeys, + data.OperationSaveAccounts: i.saveAccounts, + data.OperationFinalizedBlock: i.finalizedBlock, + } +} + +func (i *indexer) saveBlock(marshalledData []byte) error { + argsSaveBlock := &outport.ArgsSaveBlockData{} + err := i.marshaller.Unmarshal(argsSaveBlock, marshalledData) + if err != nil { + return err + } + + return i.di.SaveBlock(argsSaveBlock) +} + +func (i *indexer) revertIndexedBlock(marshalledData []byte) error { + argsRevert := &data.ArgsRevertIndexedBlock{} + err := i.marshaller.Unmarshal(argsRevert, marshalledData) + if err != nil { + return err + } + + return i.di.RevertIndexedBlock(argsRevert.Header, argsRevert.Body) +} + +func (i *indexer) saveRounds(marshalledData []byte) error { + argsRounds := &data.ArgsSaveRoundsInfo{} + err := i.marshaller.Unmarshal(argsRounds, marshalledData) + if err != nil { + return err + } + + return i.di.SaveRoundsInfo(argsRounds.RoundsInfos) +} + +func (i *indexer) saveValidatorsRating(marshalledData []byte) error { + argsValidatorsRating := &data.ArgsSaveValidatorsRating{} + err := i.marshaller.Unmarshal(argsValidatorsRating, marshalledData) + if err != nil { + return err + } + + return i.di.SaveValidatorsRating(argsValidatorsRating.IndexID, argsValidatorsRating.InfoRating) +} + +func (i *indexer) saveValidatorsPubKeys(marshalledData []byte) error { + argsValidators := &data.ArgsSaveValidatorsPubKeys{} + err := i.marshaller.Unmarshal(argsValidators, marshalledData) + if err != nil { + return err + } + + return i.di.SaveValidatorsPubKeys(argsValidators.ValidatorsPubKeys, argsValidators.Epoch) +} + +func (i *indexer) saveAccounts(marshalledData []byte) error { + argsSaveAccounts := &data.ArgsSaveAccounts{} + err := i.marshaller.Unmarshal(argsSaveAccounts, marshalledData) + if err != nil { + return err + } + + return i.di.SaveAccounts(argsSaveAccounts.BlockTimestamp, argsSaveAccounts.Acc) +} + +func (i *indexer) finalizedBlock(_ []byte) error { + return nil +} diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go new file mode 100644 index 00000000..b8566811 --- /dev/null +++ b/process/wsindexer/interface.go @@ -0,0 +1,23 @@ +package wsindexer + +import ( + "github.com/ElrondNetwork/elrond-go-core/data" + "github.com/ElrondNetwork/elrond-go-core/data/outport" +) + +type WSClient interface { + Start() + Close() +} + +type DataIndexer interface { + SaveBlock(args *outport.ArgsSaveBlockData) error + RevertIndexedBlock(header data.HeaderHandler, body data.BodyHandler) error + SaveRoundsInfo(roundsInfos []*outport.RoundInfo) error + SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error + SaveValidatorsRating(indexID string, infoRating []*outport.ValidatorRatingInfo) error + SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount) error + FinalizedBlock(headerHash []byte) error + Close() error + IsInterfaceNil() bool +} From a192b52bbd88df413527f2d09d1e61088499214a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 2 Sep 2022 15:01:10 +0300 Subject: [PATCH 35/96] move factory --- factory/indexerFactory.go | 2 +- integrationtests/utils.go | 2 +- process/{ => elasticproc}/factory/elasticProcessorFactory.go | 0 .../{ => elasticproc}/factory/elasticProcessorFactory_test.go | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename process/{ => elasticproc}/factory/elasticProcessorFactory.go (100%) rename process/{ => elasticproc}/factory/elasticProcessorFactory_test.go (100%) diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index 2600e60e..c8840c75 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -9,7 +9,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/client" "github.com/ElrondNetwork/elastic-indexer-go/client/logging" "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" - "github.com/ElrondNetwork/elastic-indexer-go/process/factory" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/factory" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/hashing" diff --git a/integrationtests/utils.go b/integrationtests/utils.go index a32aa5d2..5e33a7cd 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -10,7 +10,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc" - "github.com/ElrondNetwork/elastic-indexer-go/process/factory" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/factory" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/elastic/go-elasticsearch/v7" ) diff --git a/process/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go similarity index 100% rename from process/factory/elasticProcessorFactory.go rename to process/elasticproc/factory/elasticProcessorFactory.go diff --git a/process/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go similarity index 100% rename from process/factory/elasticProcessorFactory_test.go rename to process/elasticproc/factory/elasticProcessorFactory_test.go From 0672c96bf96f49466d9404cc74d1d07d0fec5acc Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 2 Sep 2022 15:03:41 +0300 Subject: [PATCH 36/96] small fix --- process/wsclient/client.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/process/wsclient/client.go b/process/wsclient/client.go index 1b68802a..6ddfadcb 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -10,6 +10,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/typeConverters/uint64ByteSlice" "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver" + "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/gorilla/websocket" ) @@ -28,10 +29,10 @@ var ( type client struct { urlReceive string - actions map[uint32]Handler + actions map[data.OperationType]Handler } -func NewWebSocketClient(urlReceive string, actions map[uint32]Handler) (*client, error) { +func NewWebSocketClient(urlReceive string, actions map[data.OperationType]Handler) (*client, error) { urlReceiveData := url.URL{Scheme: "ws", Host: fmt.Sprintf(urlReceive), Path: "/operations"} return &client{ @@ -119,7 +120,7 @@ func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsCo "data", payloadData.Payload, ) - function, ok := c.actions[payloadData.OperationType.Uint32()] + function, ok := c.actions[payloadData.OperationType] if !ok { log.Warn("invalid operation", "operation type", payloadData.OperationType.String()) } From 02a08b189e461fe9346b915da36852a7027718c5 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 14 Sep 2022 15:27:54 +0300 Subject: [PATCH 37/96] remove is in import db flag --- factory/indexerFactory.go | 2 - .../factory/elasticProcessorFactory.go | 6 +-- .../factory/elasticProcessorFactory_test.go | 1 - process/elasticproc/interface.go | 3 +- .../miniblocks/miniblocksProcessor.go | 17 ++++---- .../operations/operationsProcessor.go | 13 +++--- .../transactions/transactionsGrouper.go | 42 +++++++++---------- .../transactions/transactionsGrouper_test.go | 12 +++--- .../transactions/transactionsProcessor.go | 19 ++------- .../transactionsProcessor_test.go | 17 ++++---- process/wsindexer/indexer.go | 5 +++ 11 files changed, 60 insertions(+), 77 deletions(-) diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index c8840c75..33eaeaba 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -26,7 +26,6 @@ type ArgsIndexerFactory struct { Enabled bool UseKibana bool // move this in ArgSaveBlock - IsInImportDBMode bool IndexerCacheSize int Denomination int BulkRequestMaxSize int @@ -102,7 +101,6 @@ func createElasticProcessor(args *ArgsIndexerFactory) (dataindexer.ElasticProces UseKibana: args.UseKibana, DBClient: databaseClient, Denomination: args.Denomination, - IsInImportDBMode: args.IsInImportDBMode, ShardCoordinator: args.ShardCoordinator, EnabledIndexes: args.EnabledIndexes, BulkRequestMaxSize: args.BulkRequestMaxSize, diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 62057821..6cde914d 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -29,7 +29,6 @@ type ArgElasticProcessorFactory struct { EnabledIndexes []string Denomination int BulkRequestMaxSize int - IsInImportDBMode bool UseKibana bool } @@ -68,7 +67,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E return nil, err } - miniblocksProc, err := miniblocks.NewMiniblocksProcessor(arguments.ShardCoordinator.SelfId(), arguments.Hasher, arguments.Marshalizer, arguments.IsInImportDBMode) + miniblocksProc, err := miniblocks.NewMiniblocksProcessor(arguments.ShardCoordinator.SelfId(), arguments.Hasher, arguments.Marshalizer) if err != nil { return nil, err } @@ -84,7 +83,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E ShardCoordinator: arguments.ShardCoordinator, Hasher: arguments.Hasher, Marshalizer: arguments.Marshalizer, - IsInImportMode: arguments.IsInImportDBMode, } txsProc, err := transactions.NewTransactionsProcessor(argsTxsProc) if err != nil { @@ -103,7 +101,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E return nil, err } - operationsProc, err := operations.NewOperationsProcessor(arguments.IsInImportDBMode, arguments.ShardCoordinator) + operationsProc, err := operations.NewOperationsProcessor(arguments.ShardCoordinator) if err != nil { return nil, err } diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 5ac03546..77ecd82e 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -18,7 +18,6 @@ func TestCreateElasticProcessor(t *testing.T) { ShardCoordinator: &mock.ShardCoordinatorMock{}, EnabledIndexes: []string{"blocks"}, Denomination: 1, - IsInImportDBMode: false, UseKibana: false, } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index f0b408e7..2af729d0 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -66,6 +66,7 @@ type DBTransactionsHandler interface { body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, + isImportDB bool, ) *data.PreparedResults GetHexEncodedHashesForRemove(header coreData.HeaderHandler, body *block.Body) ([]string, []string) @@ -118,6 +119,6 @@ type DBLogsAndEventsHandler interface { // OperationsHandler defines the actions that an operations' handler should do type OperationsHandler interface { - ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult) ([]*data.Transaction, []*data.ScResult) + ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool) ([]*data.Transaction, []*data.ScResult) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index c5ebb242..6f60abef 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -18,10 +18,9 @@ import ( var log = logger.GetOrCreate("indexer/process/miniblocks") type miniblocksProcessor struct { - hasher hashing.Hasher - marshalier marshal.Marshalizer - selfShardID uint32 - importDBMode bool + hasher hashing.Hasher + marshalier marshal.Marshalizer + selfShardID uint32 } // NewMiniblocksProcessor will create a new instance of miniblocksProcessor @@ -29,7 +28,6 @@ func NewMiniblocksProcessor( selfShardID uint32, hasher hashing.Hasher, marshalier marshal.Marshalizer, - isImportDBMode bool, ) (*miniblocksProcessor, error) { if check.IfNil(marshalier) { return nil, dataindexer.ErrNilMarshalizer @@ -39,10 +37,9 @@ func NewMiniblocksProcessor( } return &miniblocksProcessor{ - hasher: hasher, - marshalier: marshalier, - selfShardID: selfShardID, - importDBMode: isImportDBMode, + hasher: hasher, + marshalier: marshalier, + selfShardID: selfShardID, }, nil } @@ -163,7 +160,7 @@ func (mp *miniblocksProcessor) GetMiniblocksHashesHexEncoded(header coreData.Hea } isCrossShard := miniblock.ReceiverShardID != miniblock.SenderShardID - shouldIgnore := selfShardID == miniblock.SenderShardID && mp.importDBMode && isCrossShard + shouldIgnore := selfShardID == miniblock.SenderShardID && isCrossShard if shouldIgnore { continue } diff --git a/process/elasticproc/operations/operationsProcessor.go b/process/elasticproc/operations/operationsProcessor.go index 5d58382c..5f578ec6 100644 --- a/process/elasticproc/operations/operationsProcessor.go +++ b/process/elasticproc/operations/operationsProcessor.go @@ -13,18 +13,16 @@ import ( ) type operationsProcessor struct { - importDBMode bool shardCoordinator dataindexer.ShardCoordinator } // NewOperationsProcessor will create a new instance of operationsProcessor -func NewOperationsProcessor(importDBMode bool, shardCoordinator dataindexer.ShardCoordinator) (*operationsProcessor, error) { +func NewOperationsProcessor(shardCoordinator dataindexer.ShardCoordinator) (*operationsProcessor, error) { if check.IfNil(shardCoordinator) { return nil, dataindexer.ErrNilShardCoordinator } return &operationsProcessor{ - importDBMode: importDBMode, shardCoordinator: shardCoordinator, }, nil } @@ -33,12 +31,13 @@ func NewOperationsProcessor(importDBMode bool, shardCoordinator dataindexer.Shar func (op *operationsProcessor) ProcessTransactionsAndSCRs( txs []*data.Transaction, scrs []*data.ScResult, + isImportDB bool, ) ([]*data.Transaction, []*data.ScResult) { newTxsSlice := make([]*data.Transaction, 0) newScrsSlice := make([]*data.ScResult, 0) for idx, tx := range txs { - if !op.shouldIndex(txs[idx].ReceiverShard) { + if !op.shouldIndex(txs[idx].ReceiverShard, isImportDB) { continue } @@ -49,7 +48,7 @@ func (op *operationsProcessor) ProcessTransactionsAndSCRs( } for idx := 0; idx < len(scrs); idx++ { - if !op.shouldIndex(scrs[idx].ReceiverShard) { + if !op.shouldIndex(scrs[idx].ReceiverShard, isImportDB) { continue } @@ -71,8 +70,8 @@ func (op *operationsProcessor) ProcessTransactionsAndSCRs( return newTxsSlice, newScrsSlice } -func (op *operationsProcessor) shouldIndex(destinationShardID uint32) bool { - if !op.importDBMode { +func (op *operationsProcessor) shouldIndex(destinationShardID uint32, isImportDB bool) bool { + if !isImportDB { return true } diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index b7a2be0f..99b57ae1 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -17,26 +17,20 @@ const ( ) type txsGrouper struct { - isInImportMode bool - selfShardID uint32 - txBuilder *dbTransactionBuilder - hasher hashing.Hasher - marshalizer marshal.Marshalizer + txBuilder *dbTransactionBuilder + hasher hashing.Hasher + marshalizer marshal.Marshalizer } func newTxsGrouper( txBuilder *dbTransactionBuilder, - isInImportMode bool, - selfShardID uint32, hasher hashing.Hasher, marshalizer marshal.Marshalizer, ) *txsGrouper { return &txsGrouper{ - txBuilder: txBuilder, - selfShardID: selfShardID, - isInImportMode: isInImportMode, - hasher: hasher, - marshalizer: marshalizer, + txBuilder: txBuilder, + hasher: hasher, + marshalizer: marshalizer, } } @@ -46,6 +40,7 @@ func (tg *txsGrouper) groupNormalTxs( header coreData.HeaderHandler, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, alteredAccounts data.AlteredAccountsHandler, + isImportDB bool, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) @@ -54,16 +49,17 @@ func (tg *txsGrouper) groupNormalTxs( return nil, err } + selfShardID := header.GetShardID() executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) - mbStatus := computeStatus(tg.selfShardID, mb.ReceiverShardID) + mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) for _, txHash := range executedTxHashes { dbTx, ok := tg.prepareNormalTxForDB(mbHash, mb, mbStatus, txHash, txs, header) if !ok { continue } - tg.addToAlteredAddresses(dbTx, alteredAccounts, mb, tg.selfShardID, false) - if tg.shouldIndex(mb.ReceiverShardID) { + tg.addToAlteredAddresses(dbTx, alteredAccounts, mb, selfShardID, false) + if tg.shouldIndex(mb.ReceiverShardID, isImportDB, selfShardID) { transactions[string(txHash)] = dbTx } } @@ -121,6 +117,7 @@ func (tg *txsGrouper) groupRewardsTxs( header coreData.HeaderHandler, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, alteredAccounts data.AlteredAccountsHandler, + isImportDB bool, ) (map[string]*data.Transaction, error) { rewardsTxs := make(map[string]*data.Transaction) mbHash, err := core.CalculateHash(tg.marshalizer, tg.hasher, mb) @@ -128,7 +125,8 @@ func (tg *txsGrouper) groupRewardsTxs( return nil, err } - mbStatus := computeStatus(tg.selfShardID, mb.ReceiverShardID) + selfShardID := header.GetShardID() + mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) for _, txHash := range executedTxHashes { rewardDBTx, ok := tg.prepareRewardTxForDB(mbHash, mb, mbStatus, txHash, txs, header) @@ -136,8 +134,8 @@ func (tg *txsGrouper) groupRewardsTxs( continue } - tg.addToAlteredAddresses(rewardDBTx, alteredAccounts, mb, tg.selfShardID, true) - if tg.shouldIndex(mb.ReceiverShardID) { + tg.addToAlteredAddresses(rewardDBTx, alteredAccounts, mb, selfShardID, true) + if tg.shouldIndex(mb.ReceiverShardID, isImportDB, selfShardID) { rewardsTxs[string(txHash)] = rewardDBTx } } @@ -188,7 +186,7 @@ func (tg *txsGrouper) groupInvalidTxs( continue } - tg.addToAlteredAddresses(invalidDBTx, alteredAccounts, mb, tg.selfShardID, false) + tg.addToAlteredAddresses(invalidDBTx, alteredAccounts, mb, header.GetShardID(), false) transactions[string(txHash)] = invalidDBTx } @@ -217,12 +215,12 @@ func (tg *txsGrouper) prepareInvalidTxForDB( return dbTx, true } -func (tg *txsGrouper) shouldIndex(destinationShardID uint32) bool { - if !tg.isInImportMode { +func (tg *txsGrouper) shouldIndex(destinationShardID uint32, isImportDB bool, selfShardID uint32) bool { + if !isImportDB { return true } - return tg.selfShardID == destinationShardID + return selfShardID == destinationShardID } func (tg *txsGrouper) groupReceipts(header coreData.HeaderHandler, txsPool map[string]coreData.TransactionHandlerWithGasUsedAndFee) []*data.Receipt { diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index a8d91dff..a1eceb9b 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -20,7 +20,7 @@ func TestGroupNormalTxs(t *testing.T) { parser := createDataFieldParserMock() txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) - grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -41,7 +41,7 @@ func TestGroupNormalTxs(t *testing.T) { } alteredAddresses := data.NewAlteredAccounts() - normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, alteredAddresses) + normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, alteredAddresses, false) require.Len(t, normalTxs, 2) require.Equal(t, 4, alteredAddresses.Len()) } @@ -51,7 +51,7 @@ func TestGroupRewardsTxs(t *testing.T) { parser := createDataFieldParserMock() txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) - grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -70,7 +70,7 @@ func TestGroupRewardsTxs(t *testing.T) { } alteredAddresses := data.NewAlteredAccounts() - normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, alteredAddresses) + normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, alteredAddresses, false) require.Len(t, normalTxs, 2) require.Equal(t, 2, alteredAddresses.Len()) } @@ -80,7 +80,7 @@ func TestGroupInvalidTxs(t *testing.T) { parser := createDataFieldParserMock() txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), &mock.ShardCoordinatorMock{}, parser) - grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -111,7 +111,7 @@ func TestGroupReceipts(t *testing.T) { parser := createDataFieldParserMock() txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) - grouper := newTxsGrouper(txBuilder, false, 0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 2517c184..ca819a8b 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -25,7 +25,6 @@ type ArgsTransactionProcessor struct { ShardCoordinator indexer.ShardCoordinator Hasher hashing.Hasher Marshalizer marshal.Marshalizer - IsInImportMode bool } type txsDatabaseProcessor struct { @@ -52,17 +51,11 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce return nil, err } - selfShardID := args.ShardCoordinator.SelfId() txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, args.ShardCoordinator, operationsDataParser) - txsDBGrouper := newTxsGrouper(txBuilder, args.IsInImportMode, selfShardID, args.Hasher, args.Marshalizer) + txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer) scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.ShardCoordinator, args.Marshalizer, args.Hasher, operationsDataParser) scrsDataToTxs := newScrsDataToTransactions() - if args.IsInImportMode { - log.Warn("the node is in import mode! Cross shard transactions and rewards where destination shard is " + - "not the current node's shard won't be indexed in Elastic Search") - } - return &txsDatabaseProcessor{ txBuilder: txBuilder, txsGrouper: txsDBGrouper, @@ -76,6 +69,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, + isImportDB bool, ) *data.PreparedResults { err := checkPrepareTransactionForDatabaseArguments(body, header, pool) if err != nil { @@ -100,14 +94,14 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( continue } - txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Txs, alteredAccounts) + txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Txs, alteredAccounts, isImportDB) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupNormalTxs", "error", errGroup) continue } mergeTxsMaps(normalTxs, txs) case block.RewardsBlock: - txs, errGroup := tdp.txsGrouper.groupRewardsTxs(mbIndex, mb, header, pool.Rewards, alteredAccounts) + txs, errGroup := tdp.txsGrouper.groupRewardsTxs(mbIndex, mb, header, pool.Rewards, alteredAccounts, isImportDB) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupRewardsTxs", "error", errGroup) continue @@ -174,11 +168,6 @@ func (tdp *txsDatabaseProcessor) GetHexEncodedHashesForRemove(header coreData.He continue } - if tdp.txsGrouper.isInImportMode { - // do not delete transactions on import DB - continue - } - txsHashesFromMiniblock := getTxsHashesFromMiniblockHexEncoded(miniblock) if miniblock.Type == block.SmartContractResultBlock { encodedScrsHashes = append(encodedScrsHashes, txsHashesFromMiniblock...) diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 91ee7fd0..c8bc28e0 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -27,7 +27,6 @@ func createMockArgsTxsDBProc() *ArgsTransactionProcessor { ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, - IsInImportMode: false, } return args } @@ -202,7 +201,7 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) assert.Equal(t, 7, len(results.Transactions)) } @@ -257,7 +256,7 @@ func TestRelayedTransactions(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) assert.Equal(t, 1, len(results.Transactions)) assert.Equal(t, 2, len(results.Transactions[0].SmartContractResults)) assert.Equal(t, transaction.TxStatusSuccess.String(), results.Transactions[0].Status) @@ -416,7 +415,7 @@ func TestAlteredAddresses(t *testing.T) { args.ShardCoordinator = shardCoordinator txDbProc, _ := NewTransactionsProcessor(args) - results := txDbProc.PrepareTransactionsForDatabase(body, hdr, pool) + results := txDbProc.PrepareTransactionsForDatabase(body, hdr, pool, false) for addrActual := range results.AlteredAccts.GetAll() { _, found := expectedAlteredAccounts[addrActual] @@ -498,7 +497,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) require.Len(t, results.Transactions, 1) require.Equal(t, tx1.GetGasLimit(), results.Transactions[0].GasUsed) } @@ -616,7 +615,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -670,7 +669,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -735,7 +734,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res := txDbProc.PrepareTransactionsForDatabase(body, header, pool) + res := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 2, len(res.ScResults)) @@ -761,7 +760,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res = txDbProc.PrepareTransactionsForDatabase(body, header, pool) + res = txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) require.Equal(t, "fail", res.Transactions[0].Status) require.Equal(t, 1, len(res.ScResults)) } diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index f809032d..9acf0f8b 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -93,3 +93,8 @@ func (i *indexer) saveAccounts(marshalledData []byte) error { func (i *indexer) finalizedBlock(_ []byte) error { return nil } + +// Close will close the indexer +func (i *indexer) Close() error { + return i.di.Close() +} From fc9a3d4fd34e2c34013356285c488b3e123739b1 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 15 Sep 2022 12:08:15 +0300 Subject: [PATCH 38/96] refactoring and fix unit tests --- cmd/elasticindexer/config/config.toml | 3 ++- go.mod | 2 +- go.sum | 4 ++-- mock/dbTransactionsHandlerStub.go | 2 +- mock/dispatcherStub.go | 4 +++- mock/elasticProcessorStub.go | 2 +- process/dataindexer/dataDispatcher.go | 2 +- process/dataindexer/dataDispatcher_test.go | 20 +++++++++---------- process/dataindexer/dataIndexer.go | 2 +- process/dataindexer/dataIndexer_test.go | 2 +- process/dataindexer/interface.go | 4 ++-- .../dataindexer/workItems}/interface.go | 2 +- .../workItems}/workItemAccounts.go | 0 .../workItems}/workItemAccounts_test.go | 2 +- .../dataindexer/workItems}/workItemBlock.go | 2 +- .../workItems}/workItemBlock_test.go | 2 +- .../dataindexer/workItems}/workItemRating.go | 0 .../workItems}/workItemRating_test.go | 2 +- .../workItems}/workItemRemoveBlock.go | 0 .../workItems}/workItemRemoveBlock_test.go | 2 +- .../dataindexer/workItems}/workItemRounds.go | 0 .../workItems}/workItemRounds_test.go | 2 +- .../workItems}/workItemValidators.go | 0 .../workItems}/workItemValidators_test.go | 2 +- process/elasticproc/elasticProcessor.go | 8 +++++--- process/elasticproc/elasticProcessor_test.go | 14 ++++++------- .../miniblocks/miniblocksProcessor.go | 9 ++------- .../miniblocks/miniblocksProcessor_test.go | 20 ++++++++++--------- .../elasticproc/miniblocks/serialize_test.go | 8 ++++---- .../operations/operationsProcessor_test.go | 16 +++++++-------- .../elasticproc/operations/serialize_test.go | 2 +- .../elasticproc/transactions/checkers_test.go | 1 - .../transactionsProcessor_test.go | 4 +++- 33 files changed, 73 insertions(+), 72 deletions(-) rename {workItems => process/dataindexer/workItems}/interface.go (97%) rename {workItems => process/dataindexer/workItems}/workItemAccounts.go (100%) rename {workItems => process/dataindexer/workItems}/workItemAccounts_test.go (93%) rename {workItems => process/dataindexer/workItems}/workItemBlock.go (99%) rename {workItems => process/dataindexer/workItems}/workItemBlock_test.go (98%) rename {workItems => process/dataindexer/workItems}/workItemRating.go (100%) rename {workItems => process/dataindexer/workItems}/workItemRating_test.go (94%) rename {workItems => process/dataindexer/workItems}/workItemRemoveBlock.go (100%) rename {workItems => process/dataindexer/workItems}/workItemRemoveBlock_test.go (95%) rename {workItems => process/dataindexer/workItems}/workItemRounds.go (100%) rename {workItems => process/dataindexer/workItems}/workItemRounds_test.go (92%) rename {workItems => process/dataindexer/workItems}/workItemValidators.go (100%) rename {workItems => process/dataindexer/workItems}/workItemValidators_test.go (94%) diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index 85b5b9ba..77600346 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -1,7 +1,8 @@ [config] enabled-indices = [ "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", - "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "collections" + "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", + "logs", "delegators", "operations", "collections" ] [config.elastic-cluster] use-kibana = false diff --git a/go.mod b/go.mod index 02927f99..6c863a82 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220831091042-a8722ecd7fba + github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220902143955-de36be423df8 github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 12c82167..b3de1a09 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220831091042-a8722ecd7fba h1:mRBgGpJjj6mwdDIHxMbUqoxwAdpvK1oVUmtv78Hk9xQ= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220831091042-a8722ecd7fba/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220902143955-de36be423df8 h1:DC0cgRHjv6SGzoCT5ejV0uMCFrR6Xkm3l+uY5+l4hLI= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220902143955-de36be423df8/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index 1266286f..0e6656b4 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -20,7 +20,7 @@ func (tps *DBTransactionProcessorStub) SerializeTransactionsFeeData(_ map[string } // PrepareTransactionsForDatabase - -func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) *data.PreparedResults { +func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, _ bool) *data.PreparedResults { if tps.PrepareTransactionsForDatabaseCalled != nil { return tps.PrepareTransactionsForDatabaseCalled(body, header, pool) } diff --git a/mock/dispatcherStub.go b/mock/dispatcherStub.go index 0b6bfbc9..9512e2b7 100644 --- a/mock/dispatcherStub.go +++ b/mock/dispatcherStub.go @@ -1,6 +1,8 @@ package mock -import "github.com/ElrondNetwork/elastic-indexer-go/workItems" +import ( + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" +) // DispatcherMock - type DispatcherMock struct { diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index a76abdbe..9753609f 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -87,7 +87,7 @@ func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, b } // SaveTransactions - -func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { +func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, _ bool) error { if eim.SaveTransactionsCalled != nil { return eim.SaveTransactionsCalled(body, header, pool, coreAlteredAccounts) } diff --git a/process/dataindexer/dataDispatcher.go b/process/dataindexer/dataDispatcher.go index ccf605ea..8bfa5d2f 100644 --- a/process/dataindexer/dataDispatcher.go +++ b/process/dataindexer/dataDispatcher.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/ElrondNetwork/elrond-go-core/core/atomic" "github.com/ElrondNetwork/elrond-go-core/core/check" logger "github.com/ElrondNetwork/elrond-go-logger" diff --git a/process/dataindexer/dataDispatcher_test.go b/process/dataindexer/dataDispatcher_test.go index ace85523..b1345a76 100644 --- a/process/dataindexer/dataDispatcher_test.go +++ b/process/dataindexer/dataDispatcher_test.go @@ -11,7 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + workItems2 "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" ) @@ -58,17 +58,17 @@ func TestDataDispatcher_StartIndexDataClose(t *testing.T) { return nil }, } - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) wg.Wait() require.True(t, called) - dispatcher.Add(workItems.NewItemAccounts(elasticProc, 0, nil)) + dispatcher.Add(workItems2.NewItemAccounts(elasticProc, 0, nil)) wg.Add(1) - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) - dispatcher.Add(workItems.NewItemRating(elasticProc, "", nil)) + dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems2.NewItemRating(elasticProc, "", nil)) wg.Add(1) - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) err = dispatcher.Close() require.NoError(t, err) } @@ -97,7 +97,7 @@ func TestDataDispatcher_Add(t *testing.T) { } start := time.Now() - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) wg.Wait() timePassed := time.Since(start) @@ -133,7 +133,7 @@ func TestDataDispatcher_AddWithErrorShouldRetryTheReprocessing(t *testing.T) { } start := time.Now() - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) wg.Wait() timePassed := time.Since(start) @@ -173,7 +173,7 @@ func TestDataDispatcher_Close(t *testing.T) { if count == 105 { w.Done() } - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) time.Sleep(50 * time.Millisecond) } } @@ -204,6 +204,6 @@ func TestDataDispatcher_RecoverPanic(t *testing.T) { }, } - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) dispatcher.doDataDispatch(context.Background()) } diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 57ca283c..1bec851b 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -2,7 +2,7 @@ package dataindexer import ( "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index cb93b79a..6b38902b 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index f9f0313c..47e2a194 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -4,7 +4,7 @@ import ( "math/big" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" @@ -34,7 +34,7 @@ type ElasticProcessor interface { RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error RemoveAccountsESDT(headerTimestamp uint64) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool) error SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error SaveRoundsInfo(infos []*data.RoundInfo) error SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error diff --git a/workItems/interface.go b/process/dataindexer/workItems/interface.go similarity index 97% rename from workItems/interface.go rename to process/dataindexer/workItems/interface.go index 7a3689ac..24f652c5 100644 --- a/workItems/interface.go +++ b/process/dataindexer/workItems/interface.go @@ -24,7 +24,7 @@ type saveBlockIndexer interface { txsSize int, ) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool) error } type saveRatingIndexer interface { diff --git a/workItems/workItemAccounts.go b/process/dataindexer/workItems/workItemAccounts.go similarity index 100% rename from workItems/workItemAccounts.go rename to process/dataindexer/workItems/workItemAccounts.go diff --git a/workItems/workItemAccounts_test.go b/process/dataindexer/workItems/workItemAccounts_test.go similarity index 93% rename from workItems/workItemAccounts_test.go rename to process/dataindexer/workItems/workItemAccounts_test.go index 970d5792..be528453 100644 --- a/workItems/workItemAccounts_test.go +++ b/process/dataindexer/workItems/workItemAccounts_test.go @@ -6,7 +6,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/stretchr/testify/require" ) diff --git a/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go similarity index 99% rename from workItems/workItemBlock.go rename to process/dataindexer/workItems/workItemBlock.go index 93762905..33c3004b 100644 --- a/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -87,7 +87,7 @@ func (wib *itemBlock) Save() error { err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) } - err = wib.indexer.SaveTransactions(body, wib.argsSaveBlock.Header, wib.argsSaveBlock.TransactionsPool, wib.argsSaveBlock.AlteredAccounts) + err = wib.indexer.SaveTransactions(body, wib.argsSaveBlock.Header, wib.argsSaveBlock.TransactionsPool, wib.argsSaveBlock.AlteredAccounts, wib.argsSaveBlock.IsImportDB) if err != nil { return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) diff --git a/workItems/workItemBlock_test.go b/process/dataindexer/workItems/workItemBlock_test.go similarity index 98% rename from workItems/workItemBlock_test.go rename to process/dataindexer/workItems/workItemBlock_test.go index 72a10bce..2f94cafd 100644 --- a/workItems/workItemBlock_test.go +++ b/process/dataindexer/workItems/workItemBlock_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" diff --git a/workItems/workItemRating.go b/process/dataindexer/workItems/workItemRating.go similarity index 100% rename from workItems/workItemRating.go rename to process/dataindexer/workItems/workItemRating.go diff --git a/workItems/workItemRating_test.go b/process/dataindexer/workItems/workItemRating_test.go similarity index 94% rename from workItems/workItemRating_test.go rename to process/dataindexer/workItems/workItemRating_test.go index 860e7e2a..a0fc0c4d 100644 --- a/workItems/workItemRating_test.go +++ b/process/dataindexer/workItems/workItemRating_test.go @@ -6,7 +6,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" ) diff --git a/workItems/workItemRemoveBlock.go b/process/dataindexer/workItems/workItemRemoveBlock.go similarity index 100% rename from workItems/workItemRemoveBlock.go rename to process/dataindexer/workItems/workItemRemoveBlock.go diff --git a/workItems/workItemRemoveBlock_test.go b/process/dataindexer/workItems/workItemRemoveBlock_test.go similarity index 95% rename from workItems/workItemRemoveBlock_test.go rename to process/dataindexer/workItems/workItemRemoveBlock_test.go index dc74dc62..4618486c 100644 --- a/workItems/workItemRemoveBlock_test.go +++ b/process/dataindexer/workItems/workItemRemoveBlock_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/stretchr/testify/require" diff --git a/workItems/workItemRounds.go b/process/dataindexer/workItems/workItemRounds.go similarity index 100% rename from workItems/workItemRounds.go rename to process/dataindexer/workItems/workItemRounds.go diff --git a/workItems/workItemRounds_test.go b/process/dataindexer/workItems/workItemRounds_test.go similarity index 92% rename from workItems/workItemRounds_test.go rename to process/dataindexer/workItems/workItemRounds_test.go index fd895052..1f2c05d3 100644 --- a/workItems/workItemRounds_test.go +++ b/process/dataindexer/workItems/workItemRounds_test.go @@ -6,7 +6,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" ) diff --git a/workItems/workItemValidators.go b/process/dataindexer/workItems/workItemValidators.go similarity index 100% rename from workItems/workItemValidators.go rename to process/dataindexer/workItems/workItemValidators.go diff --git a/workItems/workItemValidators_test.go b/process/dataindexer/workItems/workItemValidators_test.go similarity index 94% rename from workItems/workItemValidators_test.go rename to process/dataindexer/workItems/workItemValidators_test.go index 58296449..f2770e12 100644 --- a/workItems/workItemValidators_test.go +++ b/process/dataindexer/workItems/workItemValidators_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elastic-indexer-go/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" ) diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index c3f56de6..040d2576 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -396,10 +396,11 @@ func (ei *elasticProcessor) SaveTransactions( header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, + isImportDB bool, ) error { headerTimestamp := header.GetTimeStamp() - preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(body, header, pool) + preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(body, header, pool, isImportDB) logsData := ei.logsAndEventsProc.ExtractDataFromLogs(pool.Logs, preparedResults, headerTimestamp) buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) @@ -408,7 +409,7 @@ func (ei *elasticProcessor) SaveTransactions( return err } - err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, header, preparedResults.ScResults, buffers) + err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, header, preparedResults.ScResults, buffers, isImportDB) if err != nil { return err } @@ -538,12 +539,13 @@ func (ei *elasticProcessor) prepareAndIndexOperations( header coreData.HeaderHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice, + isImportDB bool, ) error { if !ei.isIndexEnabled(elasticIndexer.OperationsIndex) { return nil } - processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs) + processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs, isImportDB) err := ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatus, header.GetShardID(), buffSlice, elasticIndexer.OperationsIndex) if err != nil { diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 45167359..59c0af0a 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -53,7 +53,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { acp, _ := accounts.NewAccountsProcessor(&mock.PubkeyConverterMock{}, balanceConverter, 0) bp, _ := block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - mp, _ := miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) + mp, _ := miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) vp, _ := validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32), 0) args := &logsevents.ArgsLogsAndEventsProcessor{ ShardCoordinator: &mock.ShardCoordinatorMock{}, @@ -63,7 +63,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { Hasher: &mock.HasherMock{}, } lp, _ := logsevents.NewLogsAndEventsProcessor(args) - op, _ := operations.NewOperationsProcessor(false, &mock.ShardCoordinatorMock{}) + op, _ := operations.NewOperationsProcessor(&mock.ShardCoordinatorMock{}) return &ArgElasticProcessor{ DBClient: &mock.DatabaseWriterStub{}, @@ -316,7 +316,7 @@ func TestElasticProcessor_RemoveMiniblocks(t *testing.T) { }, } - args.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) + args.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) elasticProc, err := NewElasticProcessor(args) require.NoError(t, err) @@ -424,14 +424,13 @@ func TestElasticseachSaveTransactions(t *testing.T) { ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, - IsInImportMode: false, } txDbProc, _ := transactions.NewTransactionsProcessor(args) arguments.TransactionsProc = txDbProc elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) pool := &outport.Pool{Txs: txPool} - err := elasticDatabase.SaveTransactions(body, header, pool, nil) + err := elasticDatabase.SaveTransactions(body, header, pool, nil, false) require.Equal(t, localErr, err) } @@ -476,7 +475,7 @@ func TestElasticProcessor_SaveMiniblocks(t *testing.T) { }, } - arguments.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) + arguments.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) elasticProc, _ := NewElasticProcessor(arguments) header := &dataBlock.Header{} @@ -577,7 +576,6 @@ func TestElasticProcessor_RemoveTransactions(t *testing.T) { ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, - IsInImportMode: false, } txDbProc, _ := transactions.NewTransactionsProcessor(args) @@ -658,7 +656,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.enabledIndexes[dataindexer.ScResultsIndex] = struct{}{} - err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &outport.Pool{}, nil) + err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &outport.Pool{}, nil, false) require.Nil(t, err) require.False(t, called) } diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index 6f60abef..edb71fff 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -160,13 +160,8 @@ func (mp *miniblocksProcessor) GetMiniblocksHashesHexEncoded(header coreData.Hea } isCrossShard := miniblock.ReceiverShardID != miniblock.SenderShardID - shouldIgnore := selfShardID == miniblock.SenderShardID && isCrossShard - if shouldIgnore { - continue - } - - isDstMe := selfShardID == miniblock.ReceiverShardID - if isDstMe && isCrossShard { + isCrossShardOnDestination := selfShardID == miniblock.ReceiverShardID && isCrossShard + if isCrossShardOnDestination { continue } diff --git a/process/elasticproc/miniblocks/miniblocksProcessor_test.go b/process/elasticproc/miniblocks/miniblocksProcessor_test.go index aefdb6cb..91ec64b1 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor_test.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor_test.go @@ -16,20 +16,20 @@ func TestNewMiniblocksProcessor(t *testing.T) { tests := []struct { name string - args func() (uint32, hashing.Hasher, marshal.Marshalizer, bool) + args func() (uint32, hashing.Hasher, marshal.Marshalizer) exErr error }{ { name: "NilHash", - args: func() (uint32, hashing.Hasher, marshal.Marshalizer, bool) { - return 0, nil, &mock.MarshalizerMock{}, false + args: func() (uint32, hashing.Hasher, marshal.Marshalizer) { + return 0, nil, &mock.MarshalizerMock{} }, exErr: dataindexer.ErrNilHasher, }, { name: "NilMarshalizer", - args: func() (uint32, hashing.Hasher, marshal.Marshalizer, bool) { - return 0, &mock.HasherMock{}, nil, false + args: func() (uint32, hashing.Hasher, marshal.Marshalizer) { + return 0, &mock.HasherMock{}, nil }, exErr: dataindexer.ErrNilMarshalizer, }, @@ -44,7 +44,7 @@ func TestNewMiniblocksProcessor(t *testing.T) { func TestMiniblocksProcessor_PrepareDBMiniblocks(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) + mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) header := &dataBlock.Header{} body := &dataBlock.Body{ @@ -72,7 +72,7 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { t.Parallel() marshalizer := &marshal.GogoProtoMarshalizer{} - mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, marshalizer, false) + mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, marshalizer) mbhr := &dataBlock.MiniBlockHeaderReserved{ ExecutionType: dataBlock.ProcessingType(1), @@ -111,7 +111,7 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { func TestMiniblocksProcessor_GetMiniblocksHashesHexEncoded(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}, false) + mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) header := &dataBlock.Header{ MiniBlockHeaders: []dataBlock.MiniBlockHeader{ @@ -147,7 +147,7 @@ func TestMiniblocksProcessor_GetMiniblocksHashesHexEncoded(t *testing.T) { func TestMiniblocksProcessor_GetMiniblocksHashesHexEncodedImportDBMode(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(1, &mock.HasherMock{}, &mock.MarshalizerMock{}, true) + mp, _ := NewMiniblocksProcessor(1, &mock.HasherMock{}, &mock.MarshalizerMock{}) header := &dataBlock.Header{ MiniBlockHeaders: []dataBlock.MiniBlockHeader{ @@ -177,6 +177,8 @@ func TestMiniblocksProcessor_GetMiniblocksHashesHexEncodedImportDBMode(t *testin } expectedHashes := []string{ + "3acf97c324f5e8cd1e2d87de862b3105a9f08262c7914be2e186ced2a1cf1124", + "40a551b2ebc5e4b5a55e73d49ec056c72af6314606850c4d54dadfad3a7e23e5", "4a270e1ddac6b429c14c7ebccdcdd53e4f68aeebfc41552c775a7f5a5c35d06d", } miniblocksHashes := mp.GetMiniblocksHashesHexEncoded(header, body) diff --git a/process/elasticproc/miniblocks/serialize_test.go b/process/elasticproc/miniblocks/serialize_test.go index c2f09673..5f262c3d 100644 --- a/process/elasticproc/miniblocks/serialize_test.go +++ b/process/elasticproc/miniblocks/serialize_test.go @@ -12,7 +12,7 @@ import ( func TestMiniblocksProcessor_SerializeBulkMiniBlocks(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, mock.HasherMock{}, &mock.MarshalizerMock{}, false) + mp, _ := NewMiniblocksProcessor(0, mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 0, ReceiverShardID: 1}, @@ -33,7 +33,7 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocks(t *testing.T) { func TestMiniblocksProcessor_SerializeBulkMiniBlocksInDB(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, mock.HasherMock{}, &mock.MarshalizerMock{}, false) + mp, _ := NewMiniblocksProcessor(0, mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 0, ReceiverShardID: 1}, @@ -54,7 +54,7 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocksInDB(t *testing.T) { } func TestSerializeMiniblock_CrossShardNormal(t *testing.T) { - mp, _ := NewMiniblocksProcessor(1, mock.HasherMock{}, &mock.MarshalizerMock{}, false) + mp, _ := NewMiniblocksProcessor(1, mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 0, ReceiverShardID: 1, ReceiverBlockHash: "receiverBlock"}, @@ -72,7 +72,7 @@ func TestSerializeMiniblock_CrossShardNormal(t *testing.T) { } func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { - mp, _ := NewMiniblocksProcessor(1, mock.HasherMock{}, &mock.MarshalizerMock{}, false) + mp, _ := NewMiniblocksProcessor(1, mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 1, ReceiverShardID: 1, SenderBlockHash: "senderBlock", diff --git a/process/elasticproc/operations/operationsProcessor_test.go b/process/elasticproc/operations/operationsProcessor_test.go index 034d5e08..4ccaa764 100644 --- a/process/elasticproc/operations/operationsProcessor_test.go +++ b/process/elasticproc/operations/operationsProcessor_test.go @@ -13,11 +13,11 @@ import ( func TestNewOperationsProcessor(t *testing.T) { t.Parallel() - op, err := NewOperationsProcessor(false, nil) + op, err := NewOperationsProcessor(nil) require.Nil(t, op) require.Equal(t, indexer.ErrNilShardCoordinator, err) - op, err = NewOperationsProcessor(false, &mock.ShardCoordinatorMock{}) + op, err = NewOperationsProcessor(&mock.ShardCoordinatorMock{}) require.NotNil(t, op) require.Nil(t, err) } @@ -25,7 +25,7 @@ func TestNewOperationsProcessor(t *testing.T) { func TestOperationsProcessor_ProcessTransactionsAndSCRSTransactions(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(true, &mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) txs := []*data.Transaction{ {}, @@ -34,7 +34,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSTransactions(t *testing.T }, } - processedTxs, _ := op.ProcessTransactionsAndSCRs(txs, nil) + processedTxs, _ := op.ProcessTransactionsAndSCRs(txs, nil, true) require.Equal(t, []*data.Transaction{ {Type: string(transaction.TxTypeNormal)}, }, processedTxs) @@ -43,7 +43,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSTransactions(t *testing.T func TestOperationsProcessor_ProcessTransactionsAndSCRSSmartContractResults(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(true, &mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) scrs := []*data.ScResult{ {}, @@ -52,7 +52,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSSmartContractResults(t *t }, } - _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs) + _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs, true) require.Equal(t, []*data.ScResult{ {Type: string(transaction.TxTypeUnsigned), Status: transaction.TxStatusSuccess.String()}, }, processedSCRs) @@ -61,7 +61,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSSmartContractResults(t *t func TestOperationsProcessor_ShouldIgnoreSCRs(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(true, &mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) scrs := []*data.ScResult{ { @@ -77,7 +77,7 @@ func TestOperationsProcessor_ShouldIgnoreSCRs(t *testing.T) { }, } - _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs) + _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs, false) for _, scr := range processedSCRs { require.True(t, scr.CanBeIgnored) } diff --git a/process/elasticproc/operations/serialize_test.go b/process/elasticproc/operations/serialize_test.go index cc6564bb..63f2b204 100644 --- a/process/elasticproc/operations/serialize_test.go +++ b/process/elasticproc/operations/serialize_test.go @@ -11,7 +11,7 @@ import ( func TestOperationsProcessor_SerializeSCRS(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(false, &mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) scrs := []*data.ScResult{ { diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 8507b56e..e498ea28 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -22,7 +22,6 @@ func createMockArgs() *ArgsTransactionProcessor { ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, - IsInImportMode: false, } } diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index c8bc28e0..2308b7f5 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -709,7 +709,9 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, }, } - header := &block.Header{} + header := &block.Header{ + ShardID: core.MetachainShardId, + } pool := &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ "t1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ From 7d9abbd4ca7ce765e491a5956ccc3e483fdf2460 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 15 Sep 2022 17:20:02 +0300 Subject: [PATCH 39/96] fix integration tests --- factory/indexerFactory_test.go | 1 - integrationtests/accountsBalanceNftTransfer_test.go | 4 ++-- .../accountsBalanceWithLowerTimestamp_test.go | 8 ++++---- integrationtests/accountsESDTRollback_test.go | 2 +- integrationtests/accountsESDTWithTokenType_test.go | 8 ++++---- integrationtests/claimRewards_test.go | 2 +- integrationtests/collectionsIndex_test.go | 8 ++++---- integrationtests/createNFTWithTags_test.go | 6 +++--- integrationtests/esdtTransfer_test.go | 2 +- integrationtests/issueTokenAndSetRoles_test.go | 12 ++++++------ integrationtests/issueToken_test.go | 4 ++-- integrationtests/logsCrossShard_test.go | 6 +++--- integrationtests/nftIssueCreateBurn_test.go | 6 +++--- integrationtests/nftTransferCrossShard_test.go | 12 ++++++------ integrationtests/nftTransferCrossWithScCall_test.go | 2 +- integrationtests/relayedTx_test.go | 6 +++--- integrationtests/scCallIntraShard_test.go | 4 ++-- integrationtests/transactions_test.go | 2 +- integrationtests/updateNFT_test.go | 10 +++++----- integrationtests/utils.go | 3 +-- 20 files changed, 53 insertions(+), 55 deletions(-) diff --git a/factory/indexerFactory_test.go b/factory/indexerFactory_test.go index 4aa66c0a..7de90869 100644 --- a/factory/indexerFactory_test.go +++ b/factory/indexerFactory_test.go @@ -27,7 +27,6 @@ func createMockIndexerFactoryArgs() *ArgsIndexerFactory { TemplatesPath: "../testdata", EnabledIndexes: []string{"blocks", "transactions", "miniblocks", "validators", "round", "accounts", "rating"}, ShardCoordinator: &mock.ShardCoordinatorMock{}, - IsInImportDBMode: false, } } diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 0e393b1d..3774ecc7 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -72,7 +72,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids := []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} @@ -127,7 +127,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids = []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 99b2811c..2dd3ade3 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -83,7 +83,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids := []string{"6161616162626262"} @@ -105,7 +105,7 @@ func TestIndexAccountsBalance(t *testing.T) { TimeStamp: 5000, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -159,7 +159,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -190,7 +190,7 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids = []string{"6161616162626262"} diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 35fe4168..df24063d 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -86,7 +86,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { TimeStamp: 5040, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids := []string{"6161616162626262-TOKEN-eeee-02"} diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 4229098b..683710b0 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -58,7 +58,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -124,7 +124,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids = []string{"6161616162626262-SEMI-abcd-02"} @@ -200,7 +200,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids := []string{"6161616162626262-TTTT-abcd-02"} @@ -241,7 +241,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids = []string{"TTTT-abcd"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index d5bb9081..2b6edd32 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -116,7 +116,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index dbd1b4f6..845b0bdd 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -57,7 +57,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) // ################ CREATE SEMI FUNGIBLE TOKEN 1 ########################## @@ -136,7 +136,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids := []string{"61616161626262626363636363636363"} genericResponse := &GenericResponse{} @@ -166,7 +166,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 22 coreAlteredAccounts[addrForLogHex].Tokens[0].Nonce = 22 - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} @@ -215,7 +215,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 125f2910..bc11adc1 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -90,7 +90,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body := &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids := []string{"6161616162626262-DESK-abcd-01"} @@ -137,7 +137,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 2 body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) genericResponse = &GenericResponse{} @@ -185,7 +185,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) require.Nil(t, err) ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index a197f0b6..e57928fe 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -96,7 +96,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 8cc168ea..fa7ebbd2 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -58,7 +58,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -87,7 +87,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -120,7 +120,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -149,7 +149,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -198,7 +198,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -226,7 +226,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 418de4be..f2375299 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -53,7 +53,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{"SSSS-abcd"} @@ -82,7 +82,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids = []string{"SSSS-abcd"} diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 15d01b66..4b10f71e 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -53,7 +53,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) ids := []string{hex.EncodeToString([]byte("cross-log"))} @@ -94,7 +94,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) @@ -127,7 +127,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index f0655722..da62bd4f 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -56,7 +56,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{"NON-abcd"} @@ -103,7 +103,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids = []string{"NON-abcd-02"} @@ -137,7 +137,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids = []string{"NON-abcd-02"} diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 882236bf..1c72d2fe 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -82,7 +82,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -122,7 +122,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -218,7 +218,7 @@ func TestNFTTransferCrossShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -256,7 +256,7 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -324,7 +324,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -384,7 +384,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index bf087ec3..3f28d78c 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -81,7 +81,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 1df731fe..7a751dd2 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -84,7 +84,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -126,7 +126,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -217,7 +217,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 97ecf9cc..dce93e63 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -75,7 +75,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -146,7 +146,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 4589954c..f19a11ae 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -57,7 +57,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { string(txHash): tx, }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index b502ff4a..de7f02fb 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -60,7 +60,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids := []string{"NFT-abcd-0e"} @@ -87,7 +87,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) // Add URIS 2 --- results should be the same @@ -108,7 +108,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) // Update attributes 1 @@ -135,7 +135,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -163,7 +163,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil) + err = esProc.SaveTransactions(body, header, pool, nil, false) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 5e33a7cd..92d133d8 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -43,8 +43,7 @@ func CreateElasticProcessor( EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.CollectionsIndex, dataindexer.OperationsIndex}, - Denomination: 18, - IsInImportDBMode: false, + Denomination: 18, } return factory.CreateElasticProcessor(args) From 15ebfe70b57466168f10cdb82595097707090c11 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 19 Sep 2022 14:30:46 +0300 Subject: [PATCH 40/96] remove shard coordinator --- factory/indexerFactory.go | 5 -- go.mod | 2 +- go.sum | 4 +- mock/dbAccountsHandlerStub.go | 6 +-- .../elasticproc/accounts/accountsProcessor.go | 13 +++-- .../accounts/accountsProcessor_test.go | 50 +++++++++---------- process/elasticproc/elasticProcessor.go | 43 ++++++++-------- process/elasticproc/elasticProcessor_test.go | 20 ++++---- .../factory/elasticProcessorFactory.go | 9 +--- process/elasticproc/interface.go | 10 ++-- .../logsevents/fungibleESDTProcessor.go | 14 +++--- .../logsevents/fungibleESDTProcessor_test.go | 21 ++------ process/elasticproc/logsevents/interface.go | 2 + .../logsevents/logsAndEventsProcessor.go | 20 +++----- .../elasticproc/logsevents/nftsProcessor.go | 17 +++---- .../miniblocks/miniblocksProcessor.go | 11 ++-- .../miniblocks/miniblocksProcessor_test.go | 18 +++---- process/elasticproc/miniblocks/serialize.go | 7 +-- .../operations/operationsProcessor.go | 26 +++------- .../operations/operationsProcessor_test.go | 20 +++----- process/elasticproc/operations/serialize.go | 6 +-- process/elasticproc/transactions/checkers.go | 3 -- .../elasticproc/transactions/checkers_test.go | 10 ---- .../smartContractResultsProcessor.go | 28 +++++------ .../smartContractResultsProcessor_test.go | 6 +-- .../transactions/transactionDBBuilder.go | 13 ++--- .../transactions/transactionDBBuilder_test.go | 1 - .../transactions/transactionsGrouper_test.go | 8 +-- .../transactions/transactionsProcessor.go | 14 +++--- .../transactionsProcessor_test.go | 30 +++-------- 30 files changed, 176 insertions(+), 261 deletions(-) diff --git a/factory/indexerFactory.go b/factory/indexerFactory.go index 33eaeaba..5a58f8f6 100644 --- a/factory/indexerFactory.go +++ b/factory/indexerFactory.go @@ -34,7 +34,6 @@ type ArgsIndexerFactory struct { Password string TemplatesPath string EnabledIndexes []string - ShardCoordinator dataindexer.ShardCoordinator Marshalizer marshal.Marshalizer Hasher hashing.Hasher AddressPubkeyConverter core.PubkeyConverter @@ -101,7 +100,6 @@ func createElasticProcessor(args *ArgsIndexerFactory) (dataindexer.ElasticProces UseKibana: args.UseKibana, DBClient: databaseClient, Denomination: args.Denomination, - ShardCoordinator: args.ShardCoordinator, EnabledIndexes: args.EnabledIndexes, BulkRequestMaxSize: args.BulkRequestMaxSize, } @@ -128,9 +126,6 @@ func checkDataIndexerParams(arguments *ArgsIndexerFactory) error { if check.IfNil(arguments.Hasher) { return dataindexer.ErrNilHasher } - if check.IfNil(arguments.ShardCoordinator) { - return dataindexer.ErrNilShardCoordinator - } return nil } diff --git a/go.mod b/go.mod index 6c863a82..78207540 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220902143955-de36be423df8 + github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index b3de1a09..39087f68 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220902143955-de36be423df8 h1:DC0cgRHjv6SGzoCT5ejV0uMCFrR6Xkm3l+uY5+l4hLI= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220902143955-de36be423df8/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd h1:3+wgMkVre4fEC3DW25NSEp8QK2gJsk1mGBH7nR9QYRk= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= diff --git a/mock/dbAccountsHandlerStub.go b/mock/dbAccountsHandlerStub.go index f0c24068..b44070bf 100644 --- a/mock/dbAccountsHandlerStub.go +++ b/mock/dbAccountsHandlerStub.go @@ -17,17 +17,17 @@ func (dba *DBAccountsHandlerStub) GetAccounts(_ data.AlteredAccountsHandler, _ m } // PrepareRegularAccountsMap - -func (dba *DBAccountsHandlerStub) PrepareRegularAccountsMap(_ uint64, _ []*data.Account) map[string]*data.AccountInfo { +func (dba *DBAccountsHandlerStub) PrepareRegularAccountsMap(_ uint64, _ []*data.Account, _ uint32) map[string]*data.AccountInfo { return nil } // PrepareAccountsMapESDT - -func (dba *DBAccountsHandlerStub) PrepareAccountsMapESDT(_ uint64, _ []*data.AccountESDT, _ data.CountTags) (map[string]*data.AccountInfo, data.TokensHandler) { +func (dba *DBAccountsHandlerStub) PrepareAccountsMapESDT(_ uint64, _ []*data.AccountESDT, _ data.CountTags, _ uint32) (map[string]*data.AccountInfo, data.TokensHandler) { return nil, nil } // PrepareAccountsHistory - -func (dba *DBAccountsHandlerStub) PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo) map[string]*data.AccountBalanceHistory { +func (dba *DBAccountsHandlerStub) PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo, _ uint32) map[string]*data.AccountBalanceHistory { if dba.PrepareAccountsHistoryCalled != nil { return dba.PrepareAccountsHistoryCalled(timestamp, accounts) } diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index d5ffa751..2d19e3de 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -22,14 +22,12 @@ var log = logger.GetOrCreate("indexer/process/accounts") type accountsProcessor struct { addressPubkeyConverter core.PubkeyConverter balanceConverter dataindexer.BalanceConverter - shardID uint32 } // NewAccountsProcessor will create a new instance of accounts processor func NewAccountsProcessor( addressPubkeyConverter core.PubkeyConverter, balanceConverter dataindexer.BalanceConverter, - shardID uint32, ) (*accountsProcessor, error) { if check.IfNil(addressPubkeyConverter) { return nil, dataindexer.ErrNilPubkeyConverter @@ -41,7 +39,6 @@ func NewAccountsProcessor( return &accountsProcessor{ addressPubkeyConverter: addressPubkeyConverter, balanceConverter: balanceConverter, - shardID: shardID, }, nil } @@ -110,7 +107,7 @@ func notZeroBalance(balance string) bool { } // PrepareRegularAccountsMap will prepare a map of regular accounts -func (ap *accountsProcessor) PrepareRegularAccountsMap(timestamp uint64, accounts []*data.Account) map[string]*data.AccountInfo { +func (ap *accountsProcessor) PrepareRegularAccountsMap(timestamp uint64, accounts []*data.Account, shardID uint32) map[string]*data.AccountInfo { accountsMap := make(map[string]*data.AccountInfo) for _, userAccount := range accounts { address := userAccount.UserAccount.Address @@ -136,7 +133,7 @@ func (ap *accountsProcessor) PrepareRegularAccountsMap(timestamp uint64, account TotalBalanceWithStake: converters.BigIntToString(balance), TotalBalanceWithStakeNum: balanceAsFloat, Timestamp: time.Duration(timestamp), - ShardID: ap.shardID, + ShardID: shardID, } accountsMap[address] = acc @@ -150,6 +147,7 @@ func (ap *accountsProcessor) PrepareAccountsMapESDT( timestamp uint64, accounts []*data.AccountESDT, tagsCount data.CountTags, + shardID uint32, ) (map[string]*data.AccountInfo, data.TokensHandler) { tokensData := data.NewTokensInfo() accountsESDTMap := make(map[string]*data.AccountInfo) @@ -185,7 +183,7 @@ func (ap *accountsProcessor) PrepareAccountsMapESDT( IsSmartContract: core.IsSmartContractAddress(addressBytes), Data: tokenMetaData, Timestamp: time.Duration(timestamp), - ShardID: ap.shardID, + ShardID: shardID, } if acc.TokenNonce == 0 { @@ -212,6 +210,7 @@ func (ap *accountsProcessor) PrepareAccountsMapESDT( func (ap *accountsProcessor) PrepareAccountsHistory( timestamp uint64, accounts map[string]*data.AccountInfo, + shardID uint32, ) map[string]*data.AccountBalanceHistory { accountsMap := make(map[string]*data.AccountBalanceHistory) for _, userAccount := range accounts { @@ -224,7 +223,7 @@ func (ap *accountsProcessor) PrepareAccountsHistory( IsSender: userAccount.IsSender, IsSmartContract: userAccount.IsSmartContract, Identifier: converters.ComputeTokenIdentifier(userAccount.TokenName, userAccount.TokenNonce), - ShardID: ap.shardID, + ShardID: shardID, } keyInMap := fmt.Sprintf("%s-%s-%d", acc.Address, acc.Token, acc.TokenNonce) accountsMap[keyInMap] = acc diff --git a/process/elasticproc/accounts/accountsProcessor_test.go b/process/elasticproc/accounts/accountsProcessor_test.go index 4e5c3c46..92f9b732 100644 --- a/process/elasticproc/accounts/accountsProcessor_test.go +++ b/process/elasticproc/accounts/accountsProcessor_test.go @@ -26,27 +26,27 @@ func TestNewAccountsProcessor(t *testing.T) { tests := []struct { name string - argsFunc func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) + argsFunc func() (core.PubkeyConverter, dataindexer.BalanceConverter) exError error }{ { name: "NilBalanceConverter", - argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) { - return &mock.PubkeyConverterMock{}, nil, 0 + argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter) { + return &mock.PubkeyConverterMock{}, nil }, exError: dataindexer.ErrNilBalanceConverter, }, { name: "NilPubKeyConverter", - argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) { - return nil, balanceConverter, 0 + argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter) { + return nil, balanceConverter }, exError: dataindexer.ErrNilPubkeyConverter, }, { name: "ShouldWork", - argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter, uint32) { - return &mock.PubkeyConverterMock{}, balanceConverter, 0 + argsFunc: func() (core.PubkeyConverter, dataindexer.BalanceConverter) { + return &mock.PubkeyConverterMock{}, balanceConverter }, exError: nil, }, @@ -63,7 +63,7 @@ func TestNewAccountsProcessor(t *testing.T) { func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) regularAccounts, esdtAccounts := ap.GetAccounts(nil, nil) require.Len(t, regularAccounts, 0) @@ -73,16 +73,16 @@ func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { func TestAccountsProcessor_PrepareRegularAccountsMapWithNil(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) - accountsInfo := ap.PrepareRegularAccountsMap(0, nil) + accountsInfo := ap.PrepareRegularAccountsMap(0, nil, 0) require.Len(t, accountsInfo, 0) } func TestGetESDTInfo(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) tokenIdentifier := "token-001" @@ -110,7 +110,7 @@ func TestGetESDTInfo(t *testing.T) { func TestGetESDTInfoNFT(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) tokenIdentifier := "token-001" @@ -142,7 +142,7 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { t.Parallel() pubKeyConverter := mock.NewPubkeyConverterMock(32) - ap, _ := NewAccountsProcessor(pubKeyConverter, balanceConverter, 0) + ap, _ := NewAccountsProcessor(pubKeyConverter, balanceConverter) require.NotNil(t, ap) nftName := "Test-nft" @@ -192,7 +192,7 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -221,7 +221,7 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -244,7 +244,7 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) alteredAccounts := data.NewAlteredAccounts() @@ -280,10 +280,10 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { IsSender: false, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) - res := ap.PrepareRegularAccountsMap(123, []*data.Account{egldAccount}) + res := ap.PrepareRegularAccountsMap(123, []*data.Account{egldAccount}, 0) require.Equal(t, &data.AccountInfo{ Address: addr, Nonce: 1, @@ -325,7 +325,7 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { }, }, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) accountsESDT := []*data.AccountESDT{ @@ -334,7 +334,7 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { } tagsCount := tags.NewTagsCount() - res, _ := ap.PrepareAccountsMapESDT(123, accountsESDT, tagsCount) + res, _ := ap.PrepareAccountsMapESDT(123, accountsESDT, tagsCount, 0) require.Len(t, res, 2) require.Equal(t, &data.AccountInfo{ @@ -379,9 +379,9 @@ func TestAccountsProcessor_PrepareAccountsHistory(t *testing.T) { }, } - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) - res := ap.PrepareAccountsHistory(100, accounts) + res := ap.PrepareAccountsHistory(100, accounts, 0) accountBalanceHistory := res["addr1-token-112-10"] require.Equal(t, &data.AccountBalanceHistory{ Address: "addr1", @@ -400,7 +400,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("no tokens with missing data or nonce higher than 0", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) oldCreator := "old creator" tokensInfo := []*data.TokenInfo{ @@ -416,7 +416,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("error loading token, should not update metadata", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) tokensInfo := []*data.TokenInfo{ { @@ -436,7 +436,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { t.Run("should work and update metadata", func(t *testing.T) { t.Parallel() - ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter, 0) + ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) metadata0, metadata1 := &esdt.MetaData{Creator: []byte("creator 0")}, &esdt.MetaData{Creator: []byte("creator 1")} tokensInfo := []*data.TokenInfo{ diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 040d2576..65081fdb 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -45,7 +45,6 @@ type objectsMap = map[string]interface{} type ArgElasticProcessor struct { BulkRequestMaxSize int UseKibana bool - SelfShardID uint32 IndexTemplates map[string]*bytes.Buffer IndexPolicies map[string]*bytes.Buffer EnabledIndexes map[string]struct{} @@ -62,7 +61,6 @@ type ArgElasticProcessor struct { type elasticProcessor struct { bulkRequestMaxSize int - selfShardID uint32 enabledIndexes map[string]struct{} elasticClient DatabaseClientHandler accountsProc DBAccountHandler @@ -89,7 +87,6 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err blockProc: arguments.BlockProc, miniblocksProc: arguments.MiniblocksProc, transactionsProc: arguments.TransactionsProc, - selfShardID: arguments.SelfShardID, statisticsProc: arguments.StatisticsProc, validatorsProc: arguments.ValidatorsProc, logsAndEventsProc: arguments.LogsAndEventsProc, @@ -281,7 +278,7 @@ func (ei *elasticProcessor) SaveHeader( func (ei *elasticProcessor) indexEpochInfoData(header coreData.HeaderHandler, buffSlice *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.EpochInfoIndex) || - ei.selfShardID != core.MetachainShardId { + header.GetShardID() != core.MetachainShardId { return nil } @@ -343,8 +340,8 @@ func (ei *elasticProcessor) removeIfHashesNotEmpty(index string, hashes []string } // RemoveAccountsESDT will remove data from accountsesdt index and accountsesdthistory -func (ei *elasticProcessor) RemoveAccountsESDT(headerTimestamp uint64) error { - query := fmt.Sprintf(`{"query": {"bool": {"must": [{"match": {"shardID": {"query": %d,"operator": "AND"}}},{"match": {"timestamp": {"query": "%d","operator": "AND"}}}]}}}`, ei.selfShardID, headerTimestamp) +func (ei *elasticProcessor) RemoveAccountsESDT(headerTimestamp uint64, shardID uint32) error { + query := fmt.Sprintf(`{"query": {"bool": {"must": [{"match": {"shardID": {"query": %d,"operator": "AND"}}},{"match": {"timestamp": {"query": "%d","operator": "AND"}}}]}}}`, shardID, headerTimestamp) err := ei.elasticClient.DoQueryRemove( elasticIndexer.AccountsESDTIndex, bytes.NewBuffer([]byte(query)), @@ -376,7 +373,7 @@ func (ei *elasticProcessor) SaveMiniblocks(header coreData.HeaderHandler, body * } buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) - ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, miniblocksInDBMap, buffSlice, elasticIndexer.MiniblocksIndex) + ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, miniblocksInDBMap, buffSlice, elasticIndexer.MiniblocksIndex, header.GetShardID()) return ei.doBulkRequests("", buffSlice.Buffers()) } @@ -440,7 +437,7 @@ func (ei *elasticProcessor) SaveTransactions( } tagsCount := tags.NewTagsCount() - err = ei.indexAlteredAccounts(headerTimestamp, preparedResults.AlteredAccts, logsData.NFTsDataUpdates, coreAlteredAccounts, buffers, tagsCount) + err = ei.indexAlteredAccounts(headerTimestamp, preparedResults.AlteredAccts, logsData.NFTsDataUpdates, coreAlteredAccounts, buffers, tagsCount, header.GetShardID()) if err != nil { return err } @@ -545,7 +542,7 @@ func (ei *elasticProcessor) prepareAndIndexOperations( return nil } - processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs, isImportDB) + processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs, isImportDB, header.GetShardID()) err := ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatus, header.GetShardID(), buffSlice, elasticIndexer.OperationsIndex) if err != nil { @@ -608,15 +605,16 @@ func (ei *elasticProcessor) indexAlteredAccounts( coreAlteredAccounts map[string]*outport.AlteredAccount, buffSlice *data.BufferSlice, tagsCount data.CountTags, + shardID uint32, ) error { regularAccountsToIndex, accountsToIndexESDT := ei.accountsProc.GetAccounts(alteredAccounts, coreAlteredAccounts) - err := ei.saveAccounts(timestamp, regularAccountsToIndex, buffSlice) + err := ei.saveAccounts(timestamp, regularAccountsToIndex, buffSlice, shardID) if err != nil { return err } - return ei.saveAccountsESDT(timestamp, accountsToIndexESDT, updatesNFTsData, buffSlice, tagsCount) + return ei.saveAccountsESDT(timestamp, accountsToIndexESDT, updatesNFTsData, buffSlice, tagsCount, shardID) } func (ei *elasticProcessor) saveAccountsESDT( @@ -625,8 +623,9 @@ func (ei *elasticProcessor) saveAccountsESDT( updatesNFTsData []*data.NFTDataUpdate, buffSlice *data.BufferSlice, tagsCount data.CountTags, + shardID uint32, ) error { - accountsESDTMap, tokensData := ei.accountsProc.PrepareAccountsMapESDT(timestamp, wrappedAccounts, tagsCount) + accountsESDTMap, tokensData := ei.accountsProc.PrepareAccountsMapESDT(timestamp, wrappedAccounts, tagsCount, shardID) err := ei.addTokenTypeAndCurrentOwnerInAccountsESDT(tokensData, accountsESDTMap) if err != nil { return err @@ -642,7 +641,7 @@ func (ei *elasticProcessor) saveAccountsESDT( return err } - return ei.saveAccountsESDTHistory(timestamp, accountsESDTMap, buffSlice) + return ei.saveAccountsESDTHistory(timestamp, accountsESDTMap, buffSlice, shardID) } func (ei *elasticProcessor) addTokenTypeAndCurrentOwnerInAccountsESDT(tokensData data.TokensHandler, accountsESDTMap map[string]*data.AccountInfo) error { @@ -721,19 +720,19 @@ func (ei *elasticProcessor) indexNFTBurnInfo(tokensData data.TokensHandler, buff } // SaveAccounts will prepare and save information about provided accounts in elasticsearch server -func (ei *elasticProcessor) SaveAccounts(timestamp uint64, accts []*data.Account) error { +func (ei *elasticProcessor) SaveAccounts(timestamp uint64, accts []*data.Account, shardID uint32) error { buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) - return ei.saveAccounts(timestamp, accts, buffSlice) + return ei.saveAccounts(timestamp, accts, buffSlice, shardID) } -func (ei *elasticProcessor) saveAccounts(timestamp uint64, accts []*data.Account, buffSlice *data.BufferSlice) error { - accountsMap := ei.accountsProc.PrepareRegularAccountsMap(timestamp, accts) +func (ei *elasticProcessor) saveAccounts(timestamp uint64, accts []*data.Account, buffSlice *data.BufferSlice, shardID uint32) error { + accountsMap := ei.accountsProc.PrepareRegularAccountsMap(timestamp, accts, shardID) err := ei.indexAccounts(accountsMap, elasticIndexer.AccountsIndex, buffSlice) if err != nil { return err } - return ei.saveAccountsHistory(timestamp, accountsMap, buffSlice) + return ei.saveAccountsHistory(timestamp, accountsMap, buffSlice, shardID) } func (ei *elasticProcessor) indexAccounts(accountsMap map[string]*data.AccountInfo, index string, buffSlice *data.BufferSlice) error { @@ -748,22 +747,22 @@ func (ei *elasticProcessor) serializeAndIndexAccounts(accountsMap map[string]*da return ei.accountsProc.SerializeAccounts(accountsMap, buffSlice, index) } -func (ei *elasticProcessor) saveAccountsESDTHistory(timestamp uint64, accountsInfoMap map[string]*data.AccountInfo, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) saveAccountsESDTHistory(timestamp uint64, accountsInfoMap map[string]*data.AccountInfo, buffSlice *data.BufferSlice, shardID uint32) error { if !ei.isIndexEnabled(elasticIndexer.AccountsESDTHistoryIndex) { return nil } - accountsMap := ei.accountsProc.PrepareAccountsHistory(timestamp, accountsInfoMap) + accountsMap := ei.accountsProc.PrepareAccountsHistory(timestamp, accountsInfoMap, shardID) return ei.serializeAndIndexAccountsHistory(accountsMap, elasticIndexer.AccountsESDTHistoryIndex, buffSlice) } -func (ei *elasticProcessor) saveAccountsHistory(timestamp uint64, accountsInfoMap map[string]*data.AccountInfo, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) saveAccountsHistory(timestamp uint64, accountsInfoMap map[string]*data.AccountInfo, buffSlice *data.BufferSlice, shardID uint32) error { if !ei.isIndexEnabled(elasticIndexer.AccountsHistoryIndex) { return nil } - accountsMap := ei.accountsProc.PrepareAccountsHistory(timestamp, accountsInfoMap) + accountsMap := ei.accountsProc.PrepareAccountsHistory(timestamp, accountsInfoMap, shardID) return ei.serializeAndIndexAccountsHistory(accountsMap, elasticIndexer.AccountsHistoryIndex, buffSlice) } diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 59c0af0a..ef1a2eb6 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -51,19 +51,18 @@ func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, argume func createMockElasticProcessorArgs() *ArgElasticProcessor { balanceConverter, _ := converters.NewBalanceConverter(10) - acp, _ := accounts.NewAccountsProcessor(&mock.PubkeyConverterMock{}, balanceConverter, 0) + acp, _ := accounts.NewAccountsProcessor(&mock.PubkeyConverterMock{}, balanceConverter) bp, _ := block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - mp, _ := miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := miniblocks.NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) vp, _ := validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32), 0) args := &logsevents.ArgsLogsAndEventsProcessor{ - ShardCoordinator: &mock.ShardCoordinatorMock{}, PubKeyConverter: &mock.PubkeyConverterMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: balanceConverter, Hasher: &mock.HasherMock{}, } lp, _ := logsevents.NewLogsAndEventsProcessor(args) - op, _ := operations.NewOperationsProcessor(&mock.ShardCoordinatorMock{}) + op, _ := operations.NewOperationsProcessor() return &ArgElasticProcessor{ DBClient: &mock.DatabaseWriterStub{}, @@ -316,7 +315,7 @@ func TestElasticProcessor_RemoveMiniblocks(t *testing.T) { }, } - args.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + args.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticProc, err := NewElasticProcessor(args) require.NoError(t, err) @@ -421,7 +420,6 @@ func TestElasticseachSaveTransactions(t *testing.T) { args := &transactions.ArgsTransactionProcessor{ AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), - ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, } @@ -475,7 +473,7 @@ func TestElasticProcessor_SaveMiniblocks(t *testing.T) { }, } - arguments.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + arguments.MiniblocksProc, _ = miniblocks.NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticProc, _ := NewElasticProcessor(arguments) header := &dataBlock.Header{} @@ -573,7 +571,6 @@ func TestElasticProcessor_RemoveTransactions(t *testing.T) { args := &transactions.ArgsTransactionProcessor{ AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), - ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, } @@ -614,11 +611,12 @@ func TestElasticProcessor_IndexEpochInfoData(t *testing.T) { } elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) - elasticSearchProc.selfShardID = core.MetachainShardId elasticSearchProc.enabledIndexes[dataindexer.EpochInfoIndex] = struct{}{} buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - shardHeader := &dataBlock.Header{} + shardHeader := &dataBlock.Header{ + ShardID: core.MetachainShardId, + } err := elasticSearchProc.indexEpochInfoData(shardHeader, buffSlice) require.True(t, errors.Is(err, dataindexer.ErrHeaderTypeAssertion)) @@ -685,7 +683,7 @@ func TestElasticProcessor_IndexAlteredAccounts(t *testing.T) { buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) alteredAccounts := data.NewAlteredAccounts() tagsCount := tags.NewTagsCount() - err := elasticSearchProc.indexAlteredAccounts(100, alteredAccounts, nil, nil, buffSlice, tagsCount) + err := elasticSearchProc.indexAlteredAccounts(100, alteredAccounts, nil, nil, buffSlice, tagsCount, 0) require.Nil(t, err) require.True(t, called) } diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 6cde914d..675fc5c1 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -25,7 +25,6 @@ type ArgElasticProcessorFactory struct { AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter DBClient elasticproc.DatabaseClientHandler - ShardCoordinator dataindexer.ShardCoordinator EnabledIndexes []string Denomination int BulkRequestMaxSize int @@ -56,7 +55,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E accountsProc, err := accounts.NewAccountsProcessor( arguments.AddressPubkeyConverter, balanceConverter, - arguments.ShardCoordinator.SelfId(), ) if err != nil { return nil, err @@ -67,7 +65,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E return nil, err } - miniblocksProc, err := miniblocks.NewMiniblocksProcessor(arguments.ShardCoordinator.SelfId(), arguments.Hasher, arguments.Marshalizer) + miniblocksProc, err := miniblocks.NewMiniblocksProcessor(arguments.Hasher, arguments.Marshalizer) if err != nil { return nil, err } @@ -80,7 +78,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E argsTxsProc := &transactions.ArgsTransactionProcessor{ AddressPubkeyConverter: arguments.AddressPubkeyConverter, - ShardCoordinator: arguments.ShardCoordinator, Hasher: arguments.Hasher, Marshalizer: arguments.Marshalizer, } @@ -90,7 +87,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E } argsLogsAndEventsProc := &logsevents.ArgsLogsAndEventsProcessor{ - ShardCoordinator: arguments.ShardCoordinator, PubKeyConverter: arguments.AddressPubkeyConverter, Marshalizer: arguments.Marshalizer, BalanceConverter: balanceConverter, @@ -101,7 +97,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E return nil, err } - operationsProc, err := operations.NewOperationsProcessor(arguments.ShardCoordinator) + operationsProc, err := operations.NewOperationsProcessor() if err != nil { return nil, err } @@ -120,7 +116,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E UseKibana: arguments.UseKibana, IndexTemplates: indexTemplates, IndexPolicies: indexPolicies, - SelfShardID: arguments.ShardCoordinator.SelfId(), OperationsProc: operationsProc, } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 2af729d0..7ff127fe 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -31,9 +31,9 @@ type DatabaseClientHandler interface { // DBAccountHandler defines the actions that an accounts' handler should do type DBAccountHandler interface { GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) - PrepareRegularAccountsMap(timestamp uint64, accounts []*data.Account) map[string]*data.AccountInfo - PrepareAccountsMapESDT(timestamp uint64, accounts []*data.AccountESDT, tagsCount data.CountTags) (map[string]*data.AccountInfo, data.TokensHandler) - PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo) map[string]*data.AccountBalanceHistory + PrepareRegularAccountsMap(timestamp uint64, accounts []*data.Account, shardID uint32) map[string]*data.AccountInfo + PrepareAccountsMapESDT(timestamp uint64, accounts []*data.AccountESDT, tagsCount data.CountTags, shardID uint32) (map[string]*data.AccountInfo, data.TokensHandler) + PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo, shardID uint32) map[string]*data.AccountBalanceHistory PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*outport.AlteredAccount) SerializeAccountsHistory(accounts map[string]*data.AccountBalanceHistory, buffSlice *data.BufferSlice, index string) error @@ -81,7 +81,7 @@ type DBMiniblocksHandler interface { PrepareDBMiniblocks(header coreData.HeaderHandler, body *block.Body) []*data.Miniblock GetMiniblocksHashesHexEncoded(header coreData.HeaderHandler, body *block.Body) []string - SerializeBulkMiniBlocks(bulkMbs []*data.Miniblock, mbsInDB map[string]bool, buffSlice *data.BufferSlice, index string) + SerializeBulkMiniBlocks(bulkMbs []*data.Miniblock, mbsInDB map[string]bool, buffSlice *data.BufferSlice, index string, shardID uint32) } // DBStatisticsHandler defines the actions that a database statistics handler should do @@ -119,6 +119,6 @@ type DBLogsAndEventsHandler interface { // OperationsHandler defines the actions that an operations' handler should do type OperationsHandler interface { - ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool) ([]*data.Transaction, []*data.ScResult) + ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, shardID uint32) ([]*data.Transaction, []*data.ScResult) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } diff --git a/process/elasticproc/logsevents/fungibleESDTProcessor.go b/process/elasticproc/logsevents/fungibleESDTProcessor.go index bbc6733b..6745ec62 100644 --- a/process/elasticproc/logsevents/fungibleESDTProcessor.go +++ b/process/elasticproc/logsevents/fungibleESDTProcessor.go @@ -4,8 +4,8 @@ import ( "math/big" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" + "github.com/ElrondNetwork/elrond-go-core/core/sharding" coreData "github.com/ElrondNetwork/elrond-go-core/data" ) @@ -15,14 +15,12 @@ const ( type fungibleESDTProcessor struct { pubKeyConverter core.PubkeyConverter - shardCoordinator dataindexer.ShardCoordinator fungibleOperationsIdentifiers map[string]struct{} } -func newFungibleESDTProcessor(pubKeyConverter core.PubkeyConverter, shardCoordinator dataindexer.ShardCoordinator) *fungibleESDTProcessor { +func newFungibleESDTProcessor(pubKeyConverter core.PubkeyConverter) *fungibleESDTProcessor { return &fungibleESDTProcessor{ - pubKeyConverter: pubKeyConverter, - shardCoordinator: shardCoordinator, + pubKeyConverter: pubKeyConverter, fungibleOperationsIdentifiers: map[string]struct{}{ core.BuiltInFunctionESDTTransfer: {}, core.BuiltInFunctionESDTBurn: {}, @@ -55,8 +53,8 @@ func (fep *fungibleESDTProcessor) processEvent(args *argsProcessEvent) argOutput } } - selfShardID := fep.shardCoordinator.SelfId() - senderShardID := fep.shardCoordinator.ComputeId(address) + selfShardID := args.selfShardID + senderShardID := sharding.ComputeShardID(address, args.numOfShards) if senderShardID == selfShardID { fep.processEventOnSenderShard(args.event, args.accounts) } @@ -95,7 +93,7 @@ func (fep *fungibleESDTProcessor) processEventDestination( } receiverAddr := topics[3] - receiverShardID := fep.shardCoordinator.ComputeId(receiverAddr) + receiverShardID := sharding.ComputeShardID(receiverAddr, args.numOfShards) encodedReceiver := fep.pubKeyConverter.Encode(receiverAddr) if receiverShardID != selfShardID { return tokenID, valueBig.String(), "", 0 diff --git a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go index aa7ff739..9fbb94ef 100644 --- a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go +++ b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go @@ -1,7 +1,6 @@ package logsevents import ( - "bytes" "math/big" "testing" @@ -15,7 +14,7 @@ import ( func TestProcessLogsAndEventsESDT_IntraShard(t *testing.T) { t.Parallel() - fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}) + fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) event := &transaction.Event{ Address: []byte("addr"), @@ -48,14 +47,7 @@ func TestProcessLogsAndEventsESDT_CrossShardOnSource(t *testing.T) { t.Parallel() receiverAddr := []byte("receiver") - fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{ - ComputeIdCalled: func(address []byte) uint32 { - if bytes.Equal(address, receiverAddr) { - return 1 - } - return 0 - }, - }) + fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) event := &transaction.Event{ Address: []byte("addr"), @@ -89,14 +81,7 @@ func TestProcessLogsAndEventsESDT_CrossShardOnDestination(t *testing.T) { senderAddr := []byte("addr") receiverAddr := []byte("receiver") - fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{ - ComputeIdCalled: func(address []byte) uint32 { - if bytes.Equal(address, senderAddr) { - return 1 - } - return 0 - }, - }) + fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) event := &transaction.Event{ Address: senderAddr, diff --git a/process/elasticproc/logsevents/interface.go b/process/elasticproc/logsevents/interface.go index 62e9b4ad..68a42814 100644 --- a/process/elasticproc/logsevents/interface.go +++ b/process/elasticproc/logsevents/interface.go @@ -17,6 +17,8 @@ type argsProcessEvent struct { tokenRolesAndProperties *tokeninfo.TokenRolesAndProperties timestamp uint64 logAddress []byte + selfShardID uint32 + numOfShards uint32 } type argOutputProcessEvent struct { diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 86fc47ae..867d3d29 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -15,7 +15,6 @@ import ( // ArgsLogsAndEventsProcessor holds all dependencies required to create new instances of logsAndEventsProcessor type ArgsLogsAndEventsProcessor struct { - ShardCoordinator dataindexer.ShardCoordinator PubKeyConverter core.PubkeyConverter Marshalizer marshal.Marshalizer BalanceConverter dataindexer.BalanceConverter @@ -47,9 +46,6 @@ func NewLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) (*logsAndEvents } func checkArgsLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) error { - if check.IfNil(args.ShardCoordinator) { - return dataindexer.ErrNilShardCoordinator - } if check.IfNil(args.PubKeyConverter) { return dataindexer.ErrNilPubkeyConverter } @@ -67,8 +63,8 @@ func checkArgsLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) error { } func createEventsProcessors(args *ArgsLogsAndEventsProcessor) []eventsProcessor { - nftsProc := newNFTsProcessor(args.ShardCoordinator, args.PubKeyConverter, args.Marshalizer) - fungibleProc := newFungibleESDTProcessor(args.PubKeyConverter, args.ShardCoordinator) + nftsProc := newNFTsProcessor(args.PubKeyConverter, args.Marshalizer) + fungibleProc := newFungibleESDTProcessor(args.PubKeyConverter) scDeploysProc := newSCDeploysProcessor(args.PubKeyConverter) informativeProc := newInformativeLogsProcessor() updateNFTProc := newNFTsPropertiesProcessor(args.PubKeyConverter) @@ -83,13 +79,13 @@ func createEventsProcessors(args *ArgsLogsAndEventsProcessor) []eventsProcessor esdtPropProc, } - if args.ShardCoordinator.SelfId() == core.MetachainShardId { - esdtIssueProc := newESDTIssueProcessor(args.PubKeyConverter) - eventsProcs = append(eventsProcs, esdtIssueProc) + //if args.ShardCoordinator.SelfId() == core.MetachainShardId { + esdtIssueProc := newESDTIssueProcessor(args.PubKeyConverter) + eventsProcs = append(eventsProcs, esdtIssueProc) - delegatorsProcessor := newDelegatorsProcessor(args.PubKeyConverter, args.BalanceConverter) - eventsProcs = append(eventsProcs, delegatorsProcessor) - } + delegatorsProcessor := newDelegatorsProcessor(args.PubKeyConverter, args.BalanceConverter) + eventsProcs = append(eventsProcs, delegatorsProcessor) + //} return eventsProcs } diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index b91fe527..8b7a202f 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -5,9 +5,9 @@ import ( "time" "github.com/ElrondNetwork/elastic-indexer-go/data" - elasticIndexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" + "github.com/ElrondNetwork/elrond-go-core/core/sharding" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/marshal" @@ -19,19 +19,16 @@ var log = logger.GetOrCreate("indexer/process/logsevents") type nftsProcessor struct { pubKeyConverter core.PubkeyConverter nftOperationsIdentifiers map[string]struct{} - shardCoordinator elasticIndexer.ShardCoordinator marshalizer marshal.Marshalizer } func newNFTsProcessor( - shardCoordinator elasticIndexer.ShardCoordinator, pubKeyConverter core.PubkeyConverter, marshalizer marshal.Marshalizer, ) *nftsProcessor { return &nftsProcessor{ - shardCoordinator: shardCoordinator, - pubKeyConverter: pubKeyConverter, - marshalizer: marshalizer, + pubKeyConverter: pubKeyConverter, + marshalizer: marshalizer, nftOperationsIdentifiers: map[string]struct{}{ core.BuiltInFunctionESDTNFTTransfer: {}, core.BuiltInFunctionESDTNFTBurn: {}, @@ -64,8 +61,8 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv } sender := args.event.GetAddress() - senderShardID := np.shardCoordinator.ComputeId(sender) - if senderShardID == np.shardCoordinator.SelfId() { + senderShardID := sharding.ComputeShardID(sender, args.numOfShards) + if senderShardID == args.selfShardID { np.processNFTEventOnSender(args.event, args.accounts, args.tokens, args.tokensSupply, args.timestamp) } @@ -83,8 +80,8 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv receiver := args.event.GetTopics()[3] encodedReceiver := np.pubKeyConverter.Encode(topics[3]) - receiverShardID := np.shardCoordinator.ComputeId(receiver) - if receiverShardID != np.shardCoordinator.SelfId() { + receiverShardID := sharding.ComputeShardID(receiver, args.numOfShards) + if receiverShardID != args.selfShardID { return argOutputProcessEvent{ identifier: identifier, value: valueBig.String(), diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index edb71fff..32c446be 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -18,14 +18,12 @@ import ( var log = logger.GetOrCreate("indexer/process/miniblocks") type miniblocksProcessor struct { - hasher hashing.Hasher - marshalier marshal.Marshalizer - selfShardID uint32 + hasher hashing.Hasher + marshalier marshal.Marshalizer } // NewMiniblocksProcessor will create a new instance of miniblocksProcessor func NewMiniblocksProcessor( - selfShardID uint32, hasher hashing.Hasher, marshalier marshal.Marshalizer, ) (*miniblocksProcessor, error) { @@ -37,9 +35,8 @@ func NewMiniblocksProcessor( } return &miniblocksProcessor{ - hasher: hasher, - marshalier: marshalier, - selfShardID: selfShardID, + hasher: hasher, + marshalier: marshalier, }, nil } diff --git a/process/elasticproc/miniblocks/miniblocksProcessor_test.go b/process/elasticproc/miniblocks/miniblocksProcessor_test.go index 91ec64b1..7da5471b 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor_test.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor_test.go @@ -16,20 +16,20 @@ func TestNewMiniblocksProcessor(t *testing.T) { tests := []struct { name string - args func() (uint32, hashing.Hasher, marshal.Marshalizer) + args func() (hashing.Hasher, marshal.Marshalizer) exErr error }{ { name: "NilHash", - args: func() (uint32, hashing.Hasher, marshal.Marshalizer) { - return 0, nil, &mock.MarshalizerMock{} + args: func() (hashing.Hasher, marshal.Marshalizer) { + return nil, &mock.MarshalizerMock{} }, exErr: dataindexer.ErrNilHasher, }, { name: "NilMarshalizer", - args: func() (uint32, hashing.Hasher, marshal.Marshalizer) { - return 0, &mock.HasherMock{}, nil + args: func() (hashing.Hasher, marshal.Marshalizer) { + return &mock.HasherMock{}, nil }, exErr: dataindexer.ErrNilMarshalizer, }, @@ -44,7 +44,7 @@ func TestNewMiniblocksProcessor(t *testing.T) { func TestMiniblocksProcessor_PrepareDBMiniblocks(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) header := &dataBlock.Header{} body := &dataBlock.Body{ @@ -72,7 +72,7 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { t.Parallel() marshalizer := &marshal.GogoProtoMarshalizer{} - mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, marshalizer) + mp, _ := NewMiniblocksProcessor(&mock.HasherMock{}, marshalizer) mbhr := &dataBlock.MiniBlockHeaderReserved{ ExecutionType: dataBlock.ProcessingType(1), @@ -111,7 +111,7 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { func TestMiniblocksProcessor_GetMiniblocksHashesHexEncoded(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, &mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) header := &dataBlock.Header{ MiniBlockHeaders: []dataBlock.MiniBlockHeader{ @@ -147,7 +147,7 @@ func TestMiniblocksProcessor_GetMiniblocksHashesHexEncoded(t *testing.T) { func TestMiniblocksProcessor_GetMiniblocksHashesHexEncodedImportDBMode(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(1, &mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) header := &dataBlock.Header{ MiniBlockHeaders: []dataBlock.MiniBlockHeader{ diff --git a/process/elasticproc/miniblocks/serialize.go b/process/elasticproc/miniblocks/serialize.go index 995c2938..dd7874dd 100644 --- a/process/elasticproc/miniblocks/serialize.go +++ b/process/elasticproc/miniblocks/serialize.go @@ -15,9 +15,10 @@ func (mp *miniblocksProcessor) SerializeBulkMiniBlocks( existsInDb map[string]bool, buffSlice *data.BufferSlice, index string, + shardID uint32, ) { for _, mb := range bulkMbs { - meta, serializedData, err := mp.prepareMiniblockData(mb, existsInDb[mb.Hash], index) + meta, serializedData, err := mp.prepareMiniblockData(mb, existsInDb[mb.Hash], index, shardID) if err != nil { log.Warn("miniblocksProcessor.prepareMiniblockData cannot prepare miniblock data", "error", err) continue @@ -31,7 +32,7 @@ func (mp *miniblocksProcessor) SerializeBulkMiniBlocks( } } -func (mp *miniblocksProcessor) prepareMiniblockData(miniblockDB *data.Miniblock, isInDB bool, index string) ([]byte, []byte, error) { +func (mp *miniblocksProcessor) prepareMiniblockData(miniblockDB *data.Miniblock, isInDB bool, index string, shardID uint32) ([]byte, []byte, error) { mbHash := miniblockDB.Hash miniblockDB.Hash = "" @@ -44,7 +45,7 @@ func (mp *miniblocksProcessor) prepareMiniblockData(miniblockDB *data.Miniblock, // prepare data for update operation meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(mbHash), "\n")) - if mp.selfShardID == miniblockDB.SenderShardID && miniblockDB.ProcessingTypeOnDestination != block.Processed.String() { + if shardID == miniblockDB.SenderShardID && miniblockDB.ProcessingTypeOnDestination != block.Processed.String() { // prepare for update sender block hash serializedData := []byte(fmt.Sprintf(`{ "doc" : { "senderBlockHash" : "%s", "procTypeS": "%s" } }`, converters.JsonEscape(miniblockDB.SenderBlockHash), converters.JsonEscape(miniblockDB.ProcessingTypeOnSource))) diff --git a/process/elasticproc/operations/operationsProcessor.go b/process/elasticproc/operations/operationsProcessor.go index 5f578ec6..ca9abad1 100644 --- a/process/elasticproc/operations/operationsProcessor.go +++ b/process/elasticproc/operations/operationsProcessor.go @@ -5,26 +5,17 @@ import ( "strings" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" - "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data/transaction" vmcommon "github.com/ElrondNetwork/elrond-vm-common" ) type operationsProcessor struct { - shardCoordinator dataindexer.ShardCoordinator } // NewOperationsProcessor will create a new instance of operationsProcessor -func NewOperationsProcessor(shardCoordinator dataindexer.ShardCoordinator) (*operationsProcessor, error) { - if check.IfNil(shardCoordinator) { - return nil, dataindexer.ErrNilShardCoordinator - } - - return &operationsProcessor{ - shardCoordinator: shardCoordinator, - }, nil +func NewOperationsProcessor() (*operationsProcessor, error) { + return &operationsProcessor{}, nil } // ProcessTransactionsAndSCRs will prepare transactions and smart contract results to be indexed @@ -32,12 +23,13 @@ func (op *operationsProcessor) ProcessTransactionsAndSCRs( txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, + selfShardID uint32, ) ([]*data.Transaction, []*data.ScResult) { newTxsSlice := make([]*data.Transaction, 0) newScrsSlice := make([]*data.ScResult, 0) for idx, tx := range txs { - if !op.shouldIndex(txs[idx].ReceiverShard, isImportDB) { + if !op.shouldIndex(txs[idx].ReceiverShard, isImportDB, selfShardID) { continue } @@ -48,7 +40,7 @@ func (op *operationsProcessor) ProcessTransactionsAndSCRs( } for idx := 0; idx < len(scrs); idx++ { - if !op.shouldIndex(scrs[idx].ReceiverShard, isImportDB) { + if !op.shouldIndex(scrs[idx].ReceiverShard, isImportDB, selfShardID) { continue } @@ -56,9 +48,7 @@ func (op *operationsProcessor) ProcessTransactionsAndSCRs( copiedScr.Type = string(transaction.TxTypeUnsigned) setCanBeIgnoredField(&copiedScr) - - selfShard := op.shardCoordinator.SelfId() - if selfShard == copiedScr.ReceiverShard { + if selfShardID == copiedScr.ReceiverShard { copiedScr.Status = transaction.TxStatusSuccess.String() } else { copiedScr.Status = transaction.TxStatusPending.String() @@ -70,12 +60,12 @@ func (op *operationsProcessor) ProcessTransactionsAndSCRs( return newTxsSlice, newScrsSlice } -func (op *operationsProcessor) shouldIndex(destinationShardID uint32, isImportDB bool) bool { +func (op *operationsProcessor) shouldIndex(destinationShardID uint32, isImportDB bool, selfShardID uint32) bool { if !isImportDB { return true } - return op.shardCoordinator.SelfId() == destinationShardID + return selfShardID == destinationShardID } func setCanBeIgnoredField(scr *data.ScResult) { diff --git a/process/elasticproc/operations/operationsProcessor_test.go b/process/elasticproc/operations/operationsProcessor_test.go index 4ccaa764..b4f06c37 100644 --- a/process/elasticproc/operations/operationsProcessor_test.go +++ b/process/elasticproc/operations/operationsProcessor_test.go @@ -4,8 +4,6 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/mock" - indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -13,11 +11,7 @@ import ( func TestNewOperationsProcessor(t *testing.T) { t.Parallel() - op, err := NewOperationsProcessor(nil) - require.Nil(t, op) - require.Equal(t, indexer.ErrNilShardCoordinator, err) - - op, err = NewOperationsProcessor(&mock.ShardCoordinatorMock{}) + op, err := NewOperationsProcessor() require.NotNil(t, op) require.Nil(t, err) } @@ -25,7 +19,7 @@ func TestNewOperationsProcessor(t *testing.T) { func TestOperationsProcessor_ProcessTransactionsAndSCRSTransactions(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor() txs := []*data.Transaction{ {}, @@ -34,7 +28,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSTransactions(t *testing.T }, } - processedTxs, _ := op.ProcessTransactionsAndSCRs(txs, nil, true) + processedTxs, _ := op.ProcessTransactionsAndSCRs(txs, nil, true, 0) require.Equal(t, []*data.Transaction{ {Type: string(transaction.TxTypeNormal)}, }, processedTxs) @@ -43,7 +37,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSTransactions(t *testing.T func TestOperationsProcessor_ProcessTransactionsAndSCRSSmartContractResults(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor() scrs := []*data.ScResult{ {}, @@ -52,7 +46,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSSmartContractResults(t *t }, } - _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs, true) + _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs, true, 0) require.Equal(t, []*data.ScResult{ {Type: string(transaction.TxTypeUnsigned), Status: transaction.TxStatusSuccess.String()}, }, processedSCRs) @@ -61,7 +55,7 @@ func TestOperationsProcessor_ProcessTransactionsAndSCRSSmartContractResults(t *t func TestOperationsProcessor_ShouldIgnoreSCRs(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor() scrs := []*data.ScResult{ { @@ -77,7 +71,7 @@ func TestOperationsProcessor_ShouldIgnoreSCRs(t *testing.T) { }, } - _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs, false) + _, processedSCRs := op.ProcessTransactionsAndSCRs(nil, scrs, false, 0) for _, scr := range processedSCRs { require.True(t, scr.CanBeIgnored) } diff --git a/process/elasticproc/operations/serialize.go b/process/elasticproc/operations/serialize.go index 5f3e0bea..e8f9da45 100644 --- a/process/elasticproc/operations/serialize.go +++ b/process/elasticproc/operations/serialize.go @@ -9,9 +9,9 @@ import ( ) // SerializeSCRs will serialize smart contract results -func (op *operationsProcessor) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error { +func (op *operationsProcessor) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string, selfShardID uint32) error { for _, scr := range scrs { - meta, serializedData, err := op.prepareSerializedDataForAScResult(scr, index) + meta, serializedData, err := op.prepareSerializedDataForAScResult(scr, index, selfShardID) if err != nil { return err } @@ -28,6 +28,7 @@ func (op *operationsProcessor) SerializeSCRs(scrs []*data.ScResult, buffSlice *d func (op *operationsProcessor) prepareSerializedDataForAScResult( scr *data.ScResult, index string, + selfShardID uint32, ) ([]byte, []byte, error) { metaData := []byte(fmt.Sprintf(`{"update":{"_index":"%s","_id":"%s"}}%s`, index, converters.JsonEscape(scr.Hash), "\n")) marshaledSCR, err := json.Marshal(scr) @@ -35,7 +36,6 @@ func (op *operationsProcessor) prepareSerializedDataForAScResult( return nil, nil, err } - selfShardID := op.shardCoordinator.SelfId() isCrossShardOnSourceShard := scr.SenderShard != scr.ReceiverShard && scr.SenderShard == selfShardID if isCrossShardOnSourceShard { serializedData := diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 5c4074a5..b161f551 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -27,9 +27,6 @@ func checkTxsProcessorArg(args *ArgsTransactionProcessor) error { if check.IfNil(args.Marshalizer) { return elasticIndexer.ErrNilMarshalizer } - if check.IfNil(args.ShardCoordinator) { - return elasticIndexer.ErrNilShardCoordinator - } if check.IfNil(args.Hasher) { return elasticIndexer.ErrNilHasher } diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index e498ea28..e681aa8d 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -19,7 +19,6 @@ import ( func createMockArgs() *ArgsTransactionProcessor { return &ArgsTransactionProcessor{ AddressPubkeyConverter: &mock.PubkeyConverterMock{}, - ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, } @@ -42,15 +41,6 @@ func TestNewTransactionsProcessor(t *testing.T) { }, exErr: elasticIndexer.ErrNilPubkeyConverter, }, - { - name: "NilShardCoordinator", - args: func() *ArgsTransactionProcessor { - args := createMockArgs() - args.ShardCoordinator = nil - return args - }, - exErr: elasticIndexer.ErrNilShardCoordinator, - }, { name: "NilMarshalizer", args: func() *ArgsTransactionProcessor { diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 5c982284..e5f7c28f 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -7,8 +7,8 @@ import ( "time" indexerData "github.com/ElrondNetwork/elastic-indexer-go/data" - indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" + "github.com/ElrondNetwork/elrond-go-core/core/sharding" "github.com/ElrondNetwork/elrond-go-core/data" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -19,26 +19,23 @@ import ( ) type smartContractResultsProcessor struct { - pubKeyConverter core.PubkeyConverter - shardCoordinator indexer.ShardCoordinator - hasher hashing.Hasher - marshalizer marshal.Marshalizer - dataFieldParser DataFieldParser + pubKeyConverter core.PubkeyConverter + hasher hashing.Hasher + marshalizer marshal.Marshalizer + dataFieldParser DataFieldParser } func newSmartContractResultsProcessor( pubKeyConverter core.PubkeyConverter, - shardCoordinator indexer.ShardCoordinator, marshalzier marshal.Marshalizer, hasher hashing.Hasher, dataFieldParser DataFieldParser, ) *smartContractResultsProcessor { return &smartContractResultsProcessor{ - pubKeyConverter: pubKeyConverter, - shardCoordinator: shardCoordinator, - marshalizer: marshalzier, - hasher: hasher, - dataFieldParser: dataFieldParser, + pubKeyConverter: pubKeyConverter, + marshalizer: marshalzier, + hasher: hasher, + dataFieldParser: dataFieldParser, } } @@ -58,7 +55,7 @@ func (proc *smartContractResultsProcessor) processSCRs( allSCRs = append(allSCRs, indexerSCRs...) } - selfShardID := proc.shardCoordinator.SelfId() + selfShardID := header.GetShardID() for scrHash, noMBScr := range txsHandler { scr, ok := noMBScr.GetTxHandler().(*smartContractResult.SmartContractResult) if !ok { @@ -176,11 +173,12 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( func (proc *smartContractResultsProcessor) addScrsReceiverToAlteredAccounts( alteredAccounts indexerData.AlteredAccountsHandler, scrs []*indexerData.ScResult, + selfShardID uint32, + numOfShards uint32, ) { for _, scr := range scrs { receiverAddr, _ := proc.pubKeyConverter.Decode(scr.Receiver) - shardID := proc.shardCoordinator.ComputeId(receiverAddr) - if shardID != proc.shardCoordinator.SelfId() { + if selfShardID != sharding.ComputeShardID(receiverAddr, numOfShards) { continue } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index 13b016f0..50dd751b 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -30,7 +30,7 @@ func TestPrepareSmartContractResult(t *testing.T) { parser := createDataFieldParserMock() pubKeyConverter := &mock.PubkeyConverterMock{} - scrsProc := newSmartContractResultsProcessor(pubKeyConverter, &mock.ShardCoordinatorMock{}, &mock.MarshalizerMock{}, &mock.HasherMock{}, parser) + scrsProc := newSmartContractResultsProcessor(pubKeyConverter, &mock.MarshalizerMock{}, &mock.HasherMock{}, parser) nonce := uint64(10) txHash := []byte("txHash") @@ -77,7 +77,7 @@ func TestAddScrsReceiverToAlteredAccounts_ShouldWork(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - scrsProc := newSmartContractResultsProcessor(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, &mock.MarshalizerMock{}, &mock.HasherMock{}, parser) + scrsProc := newSmartContractResultsProcessor(&mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}, &mock.HasherMock{}, parser) alteredAddress := data.NewAlteredAccounts() scrs := []*data.ScResult{ @@ -88,7 +88,7 @@ func TestAddScrsReceiverToAlteredAccounts_ShouldWork(t *testing.T) { Value: "1", }, } - scrsProc.addScrsReceiverToAlteredAccounts(alteredAddress, scrs) + scrsProc.addScrsReceiverToAlteredAccounts(alteredAddress, scrs, 0, 3) require.Equal(t, 1, alteredAddress.Len()) _, ok := alteredAddress.Get("020202") diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 73c81e6a..0a50222e 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -7,8 +7,8 @@ import ( "time" "github.com/ElrondNetwork/elastic-indexer-go/data" - indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" + "github.com/ElrondNetwork/elrond-go-core/core/sharding" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/receipt" @@ -21,18 +21,15 @@ const emptyString = "" type dbTransactionBuilder struct { addressPubkeyConverter core.PubkeyConverter - shardCoordinator indexer.ShardCoordinator dataFieldParser DataFieldParser } func newTransactionDBBuilder( addressPubkeyConverter core.PubkeyConverter, - shardCoordinator indexer.ShardCoordinator, dataFieldParser DataFieldParser, ) *dbTransactionBuilder { return &dbTransactionBuilder{ addressPubkeyConverter: addressPubkeyConverter, - shardCoordinator: shardCoordinator, dataFieldParser: dataFieldParser, } } @@ -128,11 +125,15 @@ func (dtb *dbTransactionBuilder) prepareReceipt( } } -func (dtb *dbTransactionBuilder) isInSameShard(sender string) bool { +func (dtb *dbTransactionBuilder) isInSameShard( + sender string, + selfShardID uint32, + numOfShard uint32, +) bool { senderBytes, err := dtb.addressPubkeyConverter.Decode(sender) if err != nil { return false } - return dtb.shardCoordinator.ComputeId(senderBytes) == dtb.shardCoordinator.SelfId() + return sharding.ComputeShardID(senderBytes, numOfShard) == selfShardID } diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index 5d471dad..70aa2ce9 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -19,7 +19,6 @@ import ( func createCommonProcessor() dbTransactionBuilder { return dbTransactionBuilder{ addressPubkeyConverter: mock.NewPubkeyConverterMock(32), - shardCoordinator: &mock.ShardCoordinatorMock{}, dataFieldParser: createDataFieldParserMock(), } } diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index a1eceb9b..7e8d0b10 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -19,7 +19,7 @@ func TestGroupNormalTxs(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -50,7 +50,7 @@ func TestGroupRewardsTxs(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -79,7 +79,7 @@ func TestGroupInvalidTxs(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), &mock.ShardCoordinatorMock{}, parser) + txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -110,7 +110,7 @@ func TestGroupReceipts(t *testing.T) { t.Parallel() parser := createDataFieldParserMock() - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}, parser) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index ca819a8b..d1dd4681 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "github.com/ElrondNetwork/elastic-indexer-go/data" - indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -22,7 +21,6 @@ var log = logger.GetOrCreate("indexer/process/transactions") // new instances type ArgsTransactionProcessor struct { AddressPubkeyConverter core.PubkeyConverter - ShardCoordinator indexer.ShardCoordinator Hasher hashing.Hasher Marshalizer marshal.Marshalizer } @@ -42,18 +40,17 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce } argsParser := &datafield.ArgsOperationDataFieldParser{ - AddressLength: args.AddressPubkeyConverter.Len(), - Marshalizer: args.Marshalizer, - ShardCoordinator: args.ShardCoordinator, + AddressLength: args.AddressPubkeyConverter.Len(), + Marshalizer: args.Marshalizer, } operationsDataParser, err := datafield.NewOperationDataFieldParser(argsParser) if err != nil { return nil, err } - txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, args.ShardCoordinator, operationsDataParser) + txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser) txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer) - scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.ShardCoordinator, args.Marshalizer, args.Hasher, operationsDataParser) + scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.Marshalizer, args.Hasher, operationsDataParser) scrsDataToTxs := newScrsDataToTransactions() return &txsDatabaseProcessor{ @@ -70,6 +67,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( header coreData.HeaderHandler, pool *outport.Pool, isImportDB bool, + numOfShards uint32, ) *data.PreparedResults { err := checkPrepareTransactionForDatabaseArguments(body, header, pool) if err != nil { @@ -123,7 +121,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( dbReceipts := tdp.txsGrouper.groupReceipts(header, pool.Receipts) dbSCResults := tdp.scrsProc.processSCRs(body, header, pool.Scrs) - tdp.scrsProc.addScrsReceiverToAlteredAccounts(alteredAccounts, dbSCResults) + tdp.scrsProc.addScrsReceiverToAlteredAccounts(alteredAccounts, dbSCResults, header.GetShardID(), numOfShards) srcsNoTxInCurrentShard := tdp.scrsDataToTxs.attachSCRsToTransactionsAndReturnSCRsWithoutTx(normalTxs, dbSCResults) tdp.scrsDataToTxs.processTransactionsAfterSCRsWereAttached(normalTxs) diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 2308b7f5..9deabb2e 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -24,7 +24,6 @@ import ( func createMockArgsTxsDBProc() *ArgsTransactionProcessor { args := &ArgsTransactionProcessor{ AddressPubkeyConverter: mock.NewPubkeyConverterMock(10), - ShardCoordinator: &mock.ShardCoordinatorMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, } @@ -201,7 +200,7 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) assert.Equal(t, 7, len(results.Transactions)) } @@ -256,7 +255,7 @@ func TestRelayedTransactions(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) assert.Equal(t, 1, len(results.Transactions)) assert.Equal(t, 2, len(results.Transactions[0].SmartContractResults)) assert.Equal(t, transaction.TxStatusSuccess.String(), results.Transactions[0].Status) @@ -400,22 +399,10 @@ func TestAlteredAddresses(t *testing.T) { }, } - shardCoordinator := &mock.ShardCoordinatorMock{ - ComputeIdCalled: func(address []byte) uint32 { - switch string(address) { - case string(address1), string(address4), string(address5), string(address7), string(address9): - return 0 - default: - return 1 - } - }, - } - args := createMockArgsTxsDBProc() - args.ShardCoordinator = shardCoordinator txDbProc, _ := NewTransactionsProcessor(args) - results := txDbProc.PrepareTransactionsForDatabase(body, hdr, pool, false) + results := txDbProc.PrepareTransactionsForDatabase(body, hdr, pool, false, 3) for addrActual := range results.AlteredAccts.GetAll() { _, found := expectedAlteredAccounts[addrActual] @@ -497,7 +484,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) require.Len(t, results.Transactions, 1) require.Equal(t, tx1.GetGasLimit(), results.Transactions[0].GasUsed) } @@ -615,7 +602,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -669,7 +656,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) + results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -686,7 +673,6 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { args := createMockArgsTxsDBProc() pubKeyConv, _ := pubkeyConverter.NewBech32PubkeyConverter(32, log) args.AddressPubkeyConverter = pubKeyConv - args.ShardCoordinator = &mock.ShardCoordinatorMock{SelfID: core.MetachainShardId} txDbProc, _ := NewTransactionsProcessor(args) decodeBech32 := func(key string) []byte { @@ -736,7 +722,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) + res := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 2, len(res.ScResults)) @@ -762,7 +748,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res = txDbProc.PrepareTransactionsForDatabase(body, header, pool, false) + res = txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) require.Equal(t, "fail", res.Transactions[0].Status) require.Equal(t, 1, len(res.ScResults)) } From ee5ab582b1f43343ceef2f0010b90386422ace78 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 21 Sep 2022 11:28:59 +0300 Subject: [PATCH 41/96] update vm common and fixes --- go.mod | 2 +- go.sum | 5 ++--- process/elasticproc/transactions/interface.go | 2 +- .../transactions/smartContractResultsProcessor.go | 11 +++++++---- .../smartContractResultsProcessor_test.go | 9 ++++----- .../elasticproc/transactions/transactionDBBuilder.go | 3 ++- .../transactions/transactionDBBuilder_test.go | 2 +- .../elasticproc/transactions/transactionsGrouper.go | 12 ++++++++---- .../transactions/transactionsGrouper_test.go | 4 ++-- .../transactions/transactionsProcessor.go | 6 +++--- 10 files changed, 31 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 78207540..2160762e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 - github.com/ElrondNetwork/elrond-vm-common v1.3.7 + github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 39087f68..7a99b0e7 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= -github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd h1:3+wgMkVre4fEC3DW25NSEp8QK2gJsk1mGBH7nR9QYRk= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= @@ -8,8 +7,8 @@ github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5a github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= -github.com/ElrondNetwork/elrond-vm-common v1.3.7 h1:nd3pTbZD+JQfhCYX4n2QlysGbIsllGjdMU4BuFfiyUk= -github.com/ElrondNetwork/elrond-vm-common v1.3.7/go.mod h1:seROQuR7RJCoCS7mgRXVAlvjztltY1c+UroAgWr/USE= +github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 h1:LM7cObmyt3ZyGXGxE8y5K/ME4SDO/ZuSaixQpsjWEYM= +github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6/go.mod h1:7qdwuZaUr5JFiOABCVgAgiuMHEJ2B8Lv5cTTAFedZNA= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= diff --git a/process/elasticproc/transactions/interface.go b/process/elasticproc/transactions/interface.go index 85d13296..b99af5c8 100644 --- a/process/elasticproc/transactions/interface.go +++ b/process/elasticproc/transactions/interface.go @@ -4,5 +4,5 @@ import datafield "github.com/ElrondNetwork/elrond-vm-common/parsers/dataField" // DataFieldParser defines what a data field parser should be able to do type DataFieldParser interface { - Parse(dataField []byte, sender, receiver []byte) *datafield.ResponseParseData + Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index e5f7c28f..b6e962ad 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -43,6 +43,7 @@ func (proc *smartContractResultsProcessor) processSCRs( body *block.Body, header coreData.HeaderHandler, txsHandler map[string]data.TransactionHandlerWithGasUsedAndFee, + numOfShards uint32, ) []*indexerData.ScResult { allSCRs := make([]*indexerData.ScResult, 0, len(txsHandler)) for _, mb := range body.MiniBlocks { @@ -50,7 +51,7 @@ func (proc *smartContractResultsProcessor) processSCRs( continue } - indexerSCRs := proc.processSCRsFromMiniblock(header, mb, txsHandler) + indexerSCRs := proc.processSCRsFromMiniblock(header, mb, txsHandler, numOfShards) allSCRs = append(allSCRs, indexerSCRs...) } @@ -62,7 +63,7 @@ func (proc *smartContractResultsProcessor) processSCRs( continue } - indexerScr := proc.prepareSmartContractResult([]byte(scrHash), nil, scr, header, selfShardID, selfShardID, noMBScr.GetFee(), noMBScr.GetGasUsed()) + indexerScr := proc.prepareSmartContractResult([]byte(scrHash), nil, scr, header, selfShardID, selfShardID, noMBScr.GetFee(), noMBScr.GetGasUsed(), numOfShards) allSCRs = append(allSCRs, indexerScr) } @@ -74,6 +75,7 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( header coreData.HeaderHandler, mb *block.MiniBlock, scrs map[string]data.TransactionHandlerWithGasUsedAndFee, + numOfShards uint32, ) []*indexerData.ScResult { mbHash, err := core.CalculateHash(proc.marshalizer, proc.hasher, mb) if err != nil { @@ -95,7 +97,7 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( continue } - indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scr, header, mb.SenderShardID, mb.ReceiverShardID, scrHandler.GetFee(), scrHandler.GetGasUsed()) + indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scr, header, mb.SenderShardID, mb.ReceiverShardID, scrHandler.GetFee(), scrHandler.GetGasUsed(), numOfShards) indexerSCRs = append(indexerSCRs, indexerSCR) delete(scrs, string(scrHash)) @@ -113,6 +115,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( receiverShard uint32, initialTxFee *big.Int, initialTxGasUsed uint64, + numOfShards uint32, ) *indexerData.ScResult { hexEncodedMBHash := "" if len(mbHash) > 0 { @@ -133,7 +136,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( originalSenderAddr = proc.pubKeyConverter.Encode(scr.OriginalSender) } - res := proc.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr) + res := proc.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, numOfShards) return &indexerData.ScResult{ Hash: hex.EncodeToString(scrHash), diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index 50dd751b..becd2d7f 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -16,9 +16,8 @@ import ( func createDataFieldParserMock() DataFieldParser { args := &datafield.ArgsOperationDataFieldParser{ - AddressLength: 32, - Marshalizer: &mock.MarshalizerMock{}, - ShardCoordinator: &mock.ShardCoordinatorMock{}, + AddressLength: 32, + Marshalizer: &mock.MarshalizerMock{}, } parser, _ := datafield.NewOperationDataFieldParser(args) @@ -49,7 +48,7 @@ func TestPrepareSmartContractResult(t *testing.T) { header := &block.Header{TimeStamp: 100} mbHash := []byte("hash") - scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, smartContractRes, header, 0, 1, big.NewInt(0), 0) + scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, smartContractRes, header, 0, 1, big.NewInt(0), 0, 3) expectedTx := &data.ScResult{ Nonce: nonce, Hash: hex.EncodeToString([]byte(scHash)), @@ -88,7 +87,7 @@ func TestAddScrsReceiverToAlteredAccounts_ShouldWork(t *testing.T) { Value: "1", }, } - scrsProc.addScrsReceiverToAlteredAccounts(alteredAddress, scrs, 0, 3) + scrsProc.addScrsReceiverToAlteredAccounts(alteredAddress, scrs, 2, 3) require.Equal(t, 1, alteredAddress.Len()) _, ok := alteredAddress.Get("020202") diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 0a50222e..e29f7231 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -44,9 +44,10 @@ func (dtb *dbTransactionBuilder) prepareTransaction( fee *big.Int, gasUsed uint64, initialPaidFee *big.Int, + numOfShards uint32, ) *data.Transaction { isScCall := core.IsSmartContractAddress(tx.RcvAddr) - res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr) + res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr, numOfShards) return &data.Transaction{ Hash: hex.EncodeToString(txHash), diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index 70aa2ce9..951ec3c1 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -77,7 +77,7 @@ func TestGetMoveBalanceTransaction(t *testing.T) { Receivers: []string{}, } - dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500, big.NewInt(100)) + dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500, big.NewInt(100), 3) require.Equal(t, expectedTx, dbTx) } diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index 99b57ae1..f2fe2d9e 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -41,6 +41,7 @@ func (tg *txsGrouper) groupNormalTxs( txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, alteredAccounts data.AlteredAccountsHandler, isImportDB bool, + numOfShards uint32, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) @@ -53,7 +54,7 @@ func (tg *txsGrouper) groupNormalTxs( executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) for _, txHash := range executedTxHashes { - dbTx, ok := tg.prepareNormalTxForDB(mbHash, mb, mbStatus, txHash, txs, header) + dbTx, ok := tg.prepareNormalTxForDB(mbHash, mb, mbStatus, txHash, txs, header, numOfShards) if !ok { continue } @@ -95,6 +96,7 @@ func (tg *txsGrouper) prepareNormalTxForDB( txHash []byte, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, header coreData.HeaderHandler, + numOfShards uint32, ) (*data.Transaction, bool) { txHandler, okGet := txs[string(txHash)] if !okGet { @@ -106,7 +108,7 @@ func (tg *txsGrouper) prepareNormalTxForDB( return nil, false } - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, mbStatus, txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee()) + dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, mbStatus, txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee(), numOfShards) return dbTx, true } @@ -172,6 +174,7 @@ func (tg *txsGrouper) groupInvalidTxs( header coreData.HeaderHandler, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, alteredAccounts data.AlteredAccountsHandler, + numOfShards uint32, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) mbHash, err := core.CalculateHash(tg.marshalizer, tg.hasher, mb) @@ -181,7 +184,7 @@ func (tg *txsGrouper) groupInvalidTxs( executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) for _, txHash := range executedTxHashes { - invalidDBTx, ok := tg.prepareInvalidTxForDB(mbHash, mb, txHash, txs, header) + invalidDBTx, ok := tg.prepareInvalidTxForDB(mbHash, mb, txHash, txs, header, numOfShards) if !ok { continue } @@ -199,6 +202,7 @@ func (tg *txsGrouper) prepareInvalidTxForDB( txHash []byte, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, header coreData.HeaderHandler, + numOfShards uint32, ) (*data.Transaction, bool) { txHandler, okGet := txs[string(txHash)] if !okGet { @@ -210,7 +214,7 @@ func (tg *txsGrouper) prepareInvalidTxForDB( return nil, false } - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee()) + dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee(), numOfShards) return dbTx, true } diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index 7e8d0b10..d03463e4 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -41,7 +41,7 @@ func TestGroupNormalTxs(t *testing.T) { } alteredAddresses := data.NewAlteredAccounts() - normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, alteredAddresses, false) + normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, alteredAddresses, false, 3) require.Len(t, normalTxs, 2) require.Equal(t, 4, alteredAddresses.Len()) } @@ -101,7 +101,7 @@ func TestGroupInvalidTxs(t *testing.T) { } alteredAddresses := data.NewAlteredAccounts() - normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, alteredAddresses) + normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, alteredAddresses, 3) require.Len(t, normalTxs, 2) require.Equal(t, 2, alteredAddresses.Len()) } diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index d1dd4681..92347a69 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -92,7 +92,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( continue } - txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Txs, alteredAccounts, isImportDB) + txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Txs, alteredAccounts, isImportDB, numOfShards) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupNormalTxs", "error", errGroup) continue @@ -106,7 +106,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( } mergeTxsMaps(rewardsTxs, txs) case block.InvalidBlock: - txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, header, pool.Invalid, alteredAccounts) + txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, header, pool.Invalid, alteredAccounts, numOfShards) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupInvalidTxs", "error", errGroup) continue @@ -119,7 +119,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( normalTxs = tdp.setTransactionSearchOrder(normalTxs) dbReceipts := tdp.txsGrouper.groupReceipts(header, pool.Receipts) - dbSCResults := tdp.scrsProc.processSCRs(body, header, pool.Scrs) + dbSCResults := tdp.scrsProc.processSCRs(body, header, pool.Scrs, numOfShards) tdp.scrsProc.addScrsReceiverToAlteredAccounts(alteredAccounts, dbSCResults, header.GetShardID(), numOfShards) From 3f2f38b40ba7396206f3bbf38ed52b01c87c2c95 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 21 Sep 2022 13:00:00 +0300 Subject: [PATCH 42/96] fix unit tests --- factory/indexerFactory_test.go | 10 ------- integrationtests/utils.go | 2 -- mock/dbTransactionsHandlerStub.go | 2 +- mock/elasticProcessorStub.go | 6 ++--- process/dataindexer/dataDispatcher_test.go | 2 +- process/dataindexer/dataIndexer.go | 4 +-- process/dataindexer/interface.go | 8 +++--- process/dataindexer/nilIndexer.go | 2 +- process/dataindexer/workItems/interface.go | 6 ++--- .../dataindexer/workItems/workItemAccounts.go | 5 +++- .../workItems/workItemAccounts_test.go | 4 +-- .../dataindexer/workItems/workItemBlock.go | 2 +- .../workItems/workItemRemoveBlock.go | 2 +- process/elasticproc/elasticProcessor.go | 7 ++--- process/elasticproc/elasticProcessor_test.go | 4 +-- .../factory/elasticProcessorFactory_test.go | 1 - process/elasticproc/interface.go | 5 +++- .../logsevents/fungibleESDTProcessor_test.go | 26 ++++++++++++------- .../logsevents/logsAndEventsProcessor.go | 12 ++++++--- .../logsevents/logsAndEventsProcessor_test.go | 20 +++----------- .../logsevents/nftsProcessor_test.go | 26 ++++++++++++------- .../elasticproc/miniblocks/serialize_test.go | 18 ++++++------- .../elasticproc/operations/serialize_test.go | 5 ++-- 23 files changed, 88 insertions(+), 91 deletions(-) diff --git a/factory/indexerFactory_test.go b/factory/indexerFactory_test.go index 7de90869..4fb4f047 100644 --- a/factory/indexerFactory_test.go +++ b/factory/indexerFactory_test.go @@ -26,7 +26,6 @@ func createMockIndexerFactoryArgs() *ArgsIndexerFactory { ValidatorPubkeyConverter: &mock.PubkeyConverterMock{}, TemplatesPath: "../testdata", EnabledIndexes: []string{"blocks", "transactions", "miniblocks", "validators", "round", "accounts", "rating"}, - ShardCoordinator: &mock.ShardCoordinatorMock{}, } } @@ -90,15 +89,6 @@ func TestNewIndexerFactory(t *testing.T) { }, exError: dataindexer.ErrNilUrl, }, - { - name: "NilShardCoordinator", - argsFunc: func() *ArgsIndexerFactory { - args := createMockIndexerFactoryArgs() - args.ShardCoordinator = nil - return args - }, - exError: dataindexer.ErrNilShardCoordinator, - }, { name: "All arguments ok", argsFunc: func() *ArgsIndexerFactory { diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 92d133d8..0e3d54c6 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -31,7 +31,6 @@ func createESClient(url string) (elasticproc.DatabaseClientHandler, error) { // CreateElasticProcessor - func CreateElasticProcessor( esClient elasticproc.DatabaseClientHandler, - shardCoordinator dataindexer.ShardCoordinator, ) (dataindexer.ElasticProcessor, error) { args := factory.ArgElasticProcessorFactory{ Marshalizer: &mock.MarshalizerMock{}, @@ -39,7 +38,6 @@ func CreateElasticProcessor( AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32), DBClient: esClient, - ShardCoordinator: shardCoordinator, EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.CollectionsIndex, dataindexer.OperationsIndex}, diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index 0e6656b4..cbb8dd03 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -20,7 +20,7 @@ func (tps *DBTransactionProcessorStub) SerializeTransactionsFeeData(_ map[string } // PrepareTransactionsForDatabase - -func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, _ bool) *data.PreparedResults { +func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, _ bool, _ uint32) *data.PreparedResults { if tps.PrepareTransactionsForDatabaseCalled != nil { return tps.PrepareTransactionsForDatabaseCalled(body, header, pool) } diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index 9753609f..f73dbe7d 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -31,7 +31,7 @@ type ElasticProcessorStub struct { } // RemoveAccountsESDT - -func (eim *ElasticProcessorStub) RemoveAccountsESDT(headerTimestamp uint64) error { +func (eim *ElasticProcessorStub) RemoveAccountsESDT(headerTimestamp uint64, _ uint32) error { if eim.RemoveAccountsESDTCalled != nil { return eim.RemoveAccountsESDTCalled(headerTimestamp) } @@ -87,7 +87,7 @@ func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, b } // SaveTransactions - -func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, _ bool) error { +func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, _ bool, _ uint32) error { if eim.SaveTransactionsCalled != nil { return eim.SaveTransactionsCalled(body, header, pool, coreAlteredAccounts) } @@ -119,7 +119,7 @@ func (eim *ElasticProcessorStub) SaveShardValidatorsPubKeys(shardID, epoch uint3 } // SaveAccounts - -func (eim *ElasticProcessorStub) SaveAccounts(timestamp uint64, acc []*data.Account) error { +func (eim *ElasticProcessorStub) SaveAccounts(timestamp uint64, acc []*data.Account, _ uint32) error { if eim.SaveAccountsCalled != nil { return eim.SaveAccountsCalled(timestamp, acc) } diff --git a/process/dataindexer/dataDispatcher_test.go b/process/dataindexer/dataDispatcher_test.go index b1345a76..69b69173 100644 --- a/process/dataindexer/dataDispatcher_test.go +++ b/process/dataindexer/dataDispatcher_test.go @@ -63,7 +63,7 @@ func TestDataDispatcher_StartIndexDataClose(t *testing.T) { require.True(t, called) - dispatcher.Add(workItems2.NewItemAccounts(elasticProc, 0, nil)) + dispatcher.Add(workItems2.NewItemAccounts(elasticProc, 0, nil, 0)) wg.Add(1) dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) dispatcher.Add(workItems2.NewItemRating(elasticProc, "", nil)) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 1bec851b..bf86c3dc 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -136,8 +136,8 @@ func (di *dataIndexer) SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]by } // SaveAccounts will save the provided accounts -func (di *dataIndexer) SaveAccounts(timestamp uint64, accounts map[string]*outport.AlteredAccount) error { - wi := workItems.NewItemAccounts(di.elasticProcessor, timestamp, accounts) +func (di *dataIndexer) SaveAccounts(timestamp uint64, accounts map[string]*outport.AlteredAccount, shardID uint32) error { + wi := workItems.NewItemAccounts(di.elasticProcessor, timestamp, accounts, shardID) di.dispatcher.Add(wi) return nil diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 47e2a194..1c42c7d5 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -32,13 +32,13 @@ type ElasticProcessor interface { RemoveHeader(header coreData.HeaderHandler) error RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error - RemoveAccountsESDT(headerTimestamp uint64) error + RemoveAccountsESDT(headerTimestamp uint64, shardID uint32) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool, numOfShards uint32) error SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error SaveRoundsInfo(infos []*data.RoundInfo) error SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error - SaveAccounts(blockTimestamp uint64, accounts []*data.Account) error + SaveAccounts(blockTimestamp uint64, accounts []*data.Account, shardID uint32) error IsInterfaceNil() bool } @@ -68,7 +68,7 @@ type Indexer interface { SaveRoundsInfo(roundsInfos []*outport.RoundInfo) error SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error SaveValidatorsRating(indexID string, infoRating []*outport.ValidatorRatingInfo) error - SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount) error + SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount, shardID uint32) error FinalizedBlock(headerHash []byte) error Close() error IsInterfaceNil() bool diff --git a/process/dataindexer/nilIndexer.go b/process/dataindexer/nilIndexer.go index 89d24ff1..b3f54269 100644 --- a/process/dataindexer/nilIndexer.go +++ b/process/dataindexer/nilIndexer.go @@ -40,7 +40,7 @@ func (ni *NilIndexer) SaveValidatorsPubKeys(_ map[uint32][][]byte, _ uint32) err } // SaveAccounts returns nil -func (ni *NilIndexer) SaveAccounts(_ uint64, _ map[string]*outport.AlteredAccount) error { +func (ni *NilIndexer) SaveAccounts(_ uint64, _ map[string]*outport.AlteredAccount, _ uint32) error { return nil } diff --git a/process/dataindexer/workItems/interface.go b/process/dataindexer/workItems/interface.go index 24f652c5..3df8e8a0 100644 --- a/process/dataindexer/workItems/interface.go +++ b/process/dataindexer/workItems/interface.go @@ -24,7 +24,7 @@ type saveBlockIndexer interface { txsSize int, ) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool) error + SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool, numOfShards uint32) error } type saveRatingIndexer interface { @@ -35,7 +35,7 @@ type removeIndexer interface { RemoveHeader(header coreData.HeaderHandler) error RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error - RemoveAccountsESDT(headerTimestamp uint64) error + RemoveAccountsESDT(headerTimestamp uint64, shardID uint32) error } type saveRounds interface { @@ -47,5 +47,5 @@ type saveValidatorsIndexer interface { } type saveAccountsIndexer interface { - SaveAccounts(blockTimestamp uint64, accounts []*data.Account) error + SaveAccounts(blockTimestamp uint64, accounts []*data.Account, shardID uint32) error } diff --git a/process/dataindexer/workItems/workItemAccounts.go b/process/dataindexer/workItems/workItemAccounts.go index 4ad77fc2..4d87ada2 100644 --- a/process/dataindexer/workItems/workItemAccounts.go +++ b/process/dataindexer/workItems/workItemAccounts.go @@ -9,6 +9,7 @@ type itemAccounts struct { indexer saveAccountsIndexer blockTimestamp uint64 accounts map[string]*outport.AlteredAccount + shardID uint32 } // NewItemAccounts will create a new instance of itemAccounts @@ -16,11 +17,13 @@ func NewItemAccounts( indexer saveAccountsIndexer, blockTimestamp uint64, accounts map[string]*outport.AlteredAccount, + shardID uint32, ) WorkItemHandler { return &itemAccounts{ indexer: indexer, accounts: accounts, blockTimestamp: blockTimestamp, + shardID: shardID, } } @@ -34,7 +37,7 @@ func (wiv *itemAccounts) Save() error { }) } - err := wiv.indexer.SaveAccounts(wiv.blockTimestamp, accounts) + err := wiv.indexer.SaveAccounts(wiv.blockTimestamp, accounts, wiv.shardID) if err != nil { log.Warn("itemAccounts.Save", "could not index account", diff --git a/process/dataindexer/workItems/workItemAccounts_test.go b/process/dataindexer/workItems/workItemAccounts_test.go index be528453..bc6ce551 100644 --- a/process/dataindexer/workItems/workItemAccounts_test.go +++ b/process/dataindexer/workItems/workItemAccounts_test.go @@ -21,7 +21,7 @@ func TestItemAccounts_Save(t *testing.T) { }, }, 0, - make(map[string]*outport.AlteredAccount), + make(map[string]*outport.AlteredAccount), 0, ) require.False(t, itemAccounts.IsInterfaceNil()) @@ -39,7 +39,7 @@ func TestItemAccounts_SaveAccountsShouldErr(t *testing.T) { }, }, 0, - make(map[string]*outport.AlteredAccount), + make(map[string]*outport.AlteredAccount), 0, ) require.False(t, itemAccounts.IsInterfaceNil()) diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index 33c3004b..e9388d5f 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -87,7 +87,7 @@ func (wib *itemBlock) Save() error { err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) } - err = wib.indexer.SaveTransactions(body, wib.argsSaveBlock.Header, wib.argsSaveBlock.TransactionsPool, wib.argsSaveBlock.AlteredAccounts, wib.argsSaveBlock.IsImportDB) + err = wib.indexer.SaveTransactions(body, wib.argsSaveBlock.Header, wib.argsSaveBlock.TransactionsPool, wib.argsSaveBlock.AlteredAccounts, wib.argsSaveBlock.IsImportDB, wib.argsSaveBlock.NumberOfShards) if err != nil { return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) diff --git a/process/dataindexer/workItems/workItemRemoveBlock.go b/process/dataindexer/workItems/workItemRemoveBlock.go index db66f14a..2bc98014 100644 --- a/process/dataindexer/workItems/workItemRemoveBlock.go +++ b/process/dataindexer/workItems/workItemRemoveBlock.go @@ -51,5 +51,5 @@ func (wirb *itemRemoveBlock) Save() error { return err } - return wirb.indexer.RemoveAccountsESDT(wirb.headerHandler.GetTimeStamp()) + return wirb.indexer.RemoveAccountsESDT(wirb.headerHandler.GetTimeStamp(), wirb.headerHandler.GetShardID()) } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 65081fdb..b75044f1 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -394,11 +394,12 @@ func (ei *elasticProcessor) SaveTransactions( pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool, + numOfShards uint32, ) error { headerTimestamp := header.GetTimeStamp() - preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(body, header, pool, isImportDB) - logsData := ei.logsAndEventsProc.ExtractDataFromLogs(pool.Logs, preparedResults, headerTimestamp) + preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(body, header, pool, isImportDB, numOfShards) + logsData := ei.logsAndEventsProc.ExtractDataFromLogs(pool.Logs, preparedResults, headerTimestamp, header.GetShardID(), numOfShards) buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) err := ei.indexTransactions(preparedResults.Transactions, preparedResults.TxHashStatus, header, buffers) @@ -549,7 +550,7 @@ func (ei *elasticProcessor) prepareAndIndexOperations( return err } - return ei.operationsProc.SerializeSCRs(processedSCRs, buffSlice, elasticIndexer.OperationsIndex) + return ei.operationsProc.SerializeSCRs(processedSCRs, buffSlice, elasticIndexer.OperationsIndex, header.GetShardID()) } // SaveValidatorsRating will save validators rating diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index ef1a2eb6..38da2ef0 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -428,7 +428,7 @@ func TestElasticseachSaveTransactions(t *testing.T) { elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) pool := &outport.Pool{Txs: txPool} - err := elasticDatabase.SaveTransactions(body, header, pool, nil, false) + err := elasticDatabase.SaveTransactions(body, header, pool, nil, false, 3) require.Equal(t, localErr, err) } @@ -654,7 +654,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.enabledIndexes[dataindexer.ScResultsIndex] = struct{}{} - err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &outport.Pool{}, nil, false) + err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &outport.Pool{}, nil, false, 3) require.Nil(t, err) require.False(t, called) } diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 77ecd82e..f9a0cf03 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -15,7 +15,6 @@ func TestCreateElasticProcessor(t *testing.T) { AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), ValidatorPubkeyConverter: &mock.PubkeyConverterMock{}, DBClient: &mock.DatabaseWriterStub{}, - ShardCoordinator: &mock.ShardCoordinatorMock{}, EnabledIndexes: []string{"blocks"}, Denomination: 1, UseKibana: false, diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 7ff127fe..fbaea63c 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -67,6 +67,7 @@ type DBTransactionsHandler interface { header coreData.HeaderHandler, pool *outport.Pool, isImportDB bool, + numOfShards uint32, ) *data.PreparedResults GetHexEncodedHashesForRemove(header coreData.HeaderHandler, body *block.Body) ([]string, []string) @@ -103,6 +104,8 @@ type DBLogsAndEventsHandler interface { logsAndEvents []*coreData.LogData, preparedResults *data.PreparedResults, timestamp uint64, + shardID uint32, + numOfShards uint32, ) *data.PreparedLogsResults SerializeLogs(logs []*data.Logs, buffSlice *data.BufferSlice, index string) error @@ -120,5 +123,5 @@ type DBLogsAndEventsHandler interface { // OperationsHandler defines the actions that an operations' handler should do type OperationsHandler interface { ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, shardID uint32) ([]*data.Transaction, []*data.ScResult) - SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error + SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string, shardID uint32) error } diff --git a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go index 9fbb94ef..5a8d03ec 100644 --- a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go +++ b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go @@ -24,8 +24,10 @@ func TestProcessLogsAndEventsESDT_IntraShard(t *testing.T) { altered := data.NewAlteredAccounts() fungibleProc.processEvent(&argsProcessEvent{ - event: event, - accounts: altered, + event: event, + accounts: altered, + selfShardID: 2, + numOfShards: 3, }) alteredAddrSender, ok := altered.Get("61646472") @@ -46,7 +48,7 @@ func TestProcessLogsAndEventsESDT_IntraShard(t *testing.T) { func TestProcessLogsAndEventsESDT_CrossShardOnSource(t *testing.T) { t.Parallel() - receiverAddr := []byte("receiver") + receiverAddr := []byte("a") fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) event := &transaction.Event{ @@ -58,8 +60,10 @@ func TestProcessLogsAndEventsESDT_CrossShardOnSource(t *testing.T) { altered := data.NewAlteredAccounts() res := fungibleProc.processEvent(&argsProcessEvent{ - event: event, - accounts: altered, + event: event, + accounts: altered, + selfShardID: 2, + numOfShards: 3, }) require.Equal(t, "my-token", res.identifier) require.Equal(t, "100", res.value) @@ -72,14 +76,14 @@ func TestProcessLogsAndEventsESDT_CrossShardOnSource(t *testing.T) { TokenIdentifier: "my-token", }, alteredAddrSender[0]) - _, ok = altered.Get("7265636569766572") + _, ok = altered.Get("61") require.False(t, ok) } func TestProcessLogsAndEventsESDT_CrossShardOnDestination(t *testing.T) { t.Parallel() - senderAddr := []byte("addr") + senderAddr := []byte("a") receiverAddr := []byte("receiver") fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) @@ -92,8 +96,10 @@ func TestProcessLogsAndEventsESDT_CrossShardOnDestination(t *testing.T) { altered := data.NewAlteredAccounts() res := fungibleProc.processEvent(&argsProcessEvent{ - event: event, - accounts: altered, + event: event, + accounts: altered, + selfShardID: 2, + numOfShards: 3, }) require.Equal(t, "my-token", res.identifier) require.Equal(t, "100", res.value) @@ -106,6 +112,6 @@ func TestProcessLogsAndEventsESDT_CrossShardOnDestination(t *testing.T) { TokenIdentifier: "my-token", }, alteredAddrSender[0]) - _, ok = altered.Get("61646472") + _, ok = altered.Get("61") require.False(t, ok) } diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 867d3d29..ab4d36f3 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -95,6 +95,8 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( logsAndEvents []*coreData.LogData, preparedResults *data.PreparedResults, timestamp uint64, + shardID uint32, + numOfShards uint32, ) *data.PreparedLogsResults { lep.logsData = newLogsData(timestamp, preparedResults.AlteredAccts, preparedResults.Transactions, preparedResults.ScResults) @@ -104,7 +106,7 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( } events := txLog.LogHandler.GetLogEvents() - lep.processEvents(txLog.TxHash, txLog.LogHandler.GetAddress(), events) + lep.processEvents(txLog.TxHash, txLog.LogHandler.GetAddress(), events, shardID, numOfShards) } return &data.PreparedLogsResults{ @@ -118,17 +120,17 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( } } -func (lep *logsAndEventsProcessor) processEvents(logHash string, logAddress []byte, events []coreData.EventHandler) { +func (lep *logsAndEventsProcessor) processEvents(logHash string, logAddress []byte, events []coreData.EventHandler, shardID uint32, numOfShards uint32) { for _, event := range events { if check.IfNil(event) { continue } - lep.processEvent(logHash, logAddress, event) + lep.processEvent(logHash, logAddress, event, shardID, numOfShards) } } -func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byte, event coreData.EventHandler) { +func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byte, event coreData.EventHandler, shardID uint32, numOfShards uint32) { logHashHexEncoded := hex.EncodeToString([]byte(logHash)) for _, proc := range lep.eventsProcessors { res := proc.processEvent(&argsProcessEvent{ @@ -142,6 +144,8 @@ func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byt scDeploys: lep.logsData.scDeploys, txs: lep.logsData.txsMap, tokenRolesAndProperties: lep.logsData.tokenRolesAndProperties, + selfShardID: shardID, + numOfShards: numOfShards, }) if res.tokenInfo != nil { lep.logsData.tokensInfo = append(lep.logsData.tokensInfo, res.tokenInfo) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index daa2f3e6..575e0393 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -18,7 +18,6 @@ import ( func createMockArgs() *ArgsLogsAndEventsProcessor { balanceConverter, _ := converters.NewBalanceConverter(10) return &ArgsLogsAndEventsProcessor{ - ShardCoordinator: &mock.ShardCoordinatorMock{}, PubKeyConverter: &mock.PubkeyConverterMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: balanceConverter, @@ -30,13 +29,8 @@ func TestNewLogsAndEventsProcessor(t *testing.T) { t.Parallel() args := createMockArgs() - args.ShardCoordinator = nil - _, err := NewLogsAndEventsProcessor(args) - require.Equal(t, elasticIndexer.ErrNilShardCoordinator, err) - - args = createMockArgs() args.PubKeyConverter = nil - _, err = NewLogsAndEventsProcessor(args) + _, err := NewLogsAndEventsProcessor(args) require.Equal(t, elasticIndexer.ErrNilPubkeyConverter, err) args = createMockArgs() @@ -154,12 +148,9 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) args := createMockArgs() balanceConverter, _ := converters.NewBalanceConverter(10) args.BalanceConverter = balanceConverter - args.ShardCoordinator = &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } proc, _ := NewLogsAndEventsProcessor(args) - resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000) + resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000, 0, 3) require.NotNil(t, resLogs.Tokens) require.True(t, res.Transactions[0].HasOperations) require.True(t, res.ScResults[0].HasOperations) @@ -235,7 +226,7 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { Hash: "747848617368", OriginalTxHash: "orignalHash", }, - }}, 1234) + }}, 1234, 0, 3) logsDB := proc.PrepareLogsForDB(logsAndEventsSlice, 1234) require.Equal(t, &data.Logs{ @@ -289,12 +280,9 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsNFTBurn(t *testing.T) { args := createMockArgs() balanceConverter, _ := converters.NewBalanceConverter(10) args.BalanceConverter = balanceConverter - args.ShardCoordinator = &mock.ShardCoordinatorMock{ - SelfID: 0, - } proc, _ := NewLogsAndEventsProcessor(args) - resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000) + resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000, 2, 3) require.Equal(t, 1, resLogs.TokensSupply.Len()) tokensSupply := resLogs.TokensSupply.GetAll() diff --git a/process/elasticproc/logsevents/nftsProcessor_test.go b/process/elasticproc/logsevents/nftsProcessor_test.go index 9cc8bb6c..0d0a7d64 100644 --- a/process/elasticproc/logsevents/nftsProcessor_test.go +++ b/process/elasticproc/logsevents/nftsProcessor_test.go @@ -31,16 +31,18 @@ func TestNftsProcessor_processLogAndEventsNFTs(t *testing.T) { Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(nonce).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, } - nftsProc := newNFTsProcessor(&mock.ShardCoordinatorMock{}, &mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) + nftsProc := newNFTsProcessor(&mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) altered := data.NewAlteredAccounts() tokensCreateInfo := data.NewTokensInfo() res := nftsProc.processEvent(&argsProcessEvent{ - event: event, - accounts: altered, - tokens: tokensCreateInfo, - timestamp: 1000, + event: event, + accounts: altered, + tokens: tokensCreateInfo, + timestamp: 1000, + selfShardID: 2, + numOfShards: 3, }) require.Equal(t, "my-token-13", res.identifier) require.Equal(t, "1", res.value) @@ -72,7 +74,7 @@ func TestNftsProcessor_processLogAndEventsNFTs_TransferNFT(t *testing.T) { t.Parallel() nonce := uint64(19) - nftsProc := newNFTsProcessor(&mock.ShardCoordinatorMock{}, &mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) + nftsProc := newNFTsProcessor(&mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) events := &transaction.Event{ Address: []byte("addr"), @@ -83,9 +85,11 @@ func TestNftsProcessor_processLogAndEventsNFTs_TransferNFT(t *testing.T) { altered := data.NewAlteredAccounts() res := nftsProc.processEvent(&argsProcessEvent{ - event: events, - accounts: altered, - timestamp: 10000, + event: events, + accounts: altered, + timestamp: 10000, + selfShardID: 2, + numOfShards: 3, }) require.Equal(t, "my-token-13", res.identifier) require.Equal(t, "1", res.value) @@ -112,7 +116,7 @@ func TestNftsProcessor_processLogAndEventsNFTs_Wipe(t *testing.T) { t.Parallel() nonce := uint64(20) - nftsProc := newNFTsProcessor(&mock.ShardCoordinatorMock{}, &mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) + nftsProc := newNFTsProcessor(&mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) events := &transaction.Event{ Address: []byte("addr"), @@ -127,6 +131,8 @@ func TestNftsProcessor_processLogAndEventsNFTs_Wipe(t *testing.T) { accounts: altered, timestamp: 10000, tokensSupply: data.NewTokensInfo(), + numOfShards: 3, + selfShardID: 2, }) require.Equal(t, "nft-0123-14", res.identifier) require.Equal(t, "1", res.value) diff --git a/process/elasticproc/miniblocks/serialize_test.go b/process/elasticproc/miniblocks/serialize_test.go index 5f262c3d..60c170a0 100644 --- a/process/elasticproc/miniblocks/serialize_test.go +++ b/process/elasticproc/miniblocks/serialize_test.go @@ -12,7 +12,7 @@ import ( func TestMiniblocksProcessor_SerializeBulkMiniBlocks(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := NewMiniblocksProcessor(mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 0, ReceiverShardID: 1}, @@ -20,7 +20,7 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocks(t *testing.T) { } buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - mp.SerializeBulkMiniBlocks(miniblocks, nil, buffSlice, "miniblocks") + mp.SerializeBulkMiniBlocks(miniblocks, nil, buffSlice, "miniblocks", 0) expectedBuff := `{ "index" : { "_index":"miniblocks", "_id" : "h1"} } {"senderShard":0,"receiverShard":1,"type":"","timestamp":0} @@ -33,7 +33,7 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocks(t *testing.T) { func TestMiniblocksProcessor_SerializeBulkMiniBlocksInDB(t *testing.T) { t.Parallel() - mp, _ := NewMiniblocksProcessor(0, mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := NewMiniblocksProcessor(mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 0, ReceiverShardID: 1}, @@ -43,7 +43,7 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocksInDB(t *testing.T) { buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ "h1": true, - }, buffSlice, "miniblocks") + }, buffSlice, "miniblocks", 0) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } { "doc" : { "senderBlockHash" : "", "procTypeS": "" } } @@ -54,7 +54,7 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocksInDB(t *testing.T) { } func TestSerializeMiniblock_CrossShardNormal(t *testing.T) { - mp, _ := NewMiniblocksProcessor(1, mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := NewMiniblocksProcessor(mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 0, ReceiverShardID: 1, ReceiverBlockHash: "receiverBlock"}, @@ -63,7 +63,7 @@ func TestSerializeMiniblock_CrossShardNormal(t *testing.T) { buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ "h1": true, - }, buffSlice, "miniblocks") + }, buffSlice, "miniblocks", 1) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } { "doc" : { "receiverBlockHash" : "receiverBlock", "procTypeD": "" } } @@ -72,7 +72,7 @@ func TestSerializeMiniblock_CrossShardNormal(t *testing.T) { } func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { - mp, _ := NewMiniblocksProcessor(1, mock.HasherMock{}, &mock.MarshalizerMock{}) + mp, _ := NewMiniblocksProcessor(mock.HasherMock{}, &mock.MarshalizerMock{}) miniblocks := []*data.Miniblock{ {Hash: "h1", SenderShardID: 1, ReceiverShardID: 1, SenderBlockHash: "senderBlock", @@ -82,7 +82,7 @@ func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ "h1": false, - }, buffSlice, "miniblocks") + }, buffSlice, "miniblocks", 1) expectedBuff := `{ "index" : { "_index":"miniblocks", "_id" : "h1"} } {"senderShard":1,"receiverShard":1,"senderBlockHash":"senderBlock","type":"","procTypeS":"Scheduled","timestamp":0} @@ -97,7 +97,7 @@ func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { buffSlice = data.NewBufferSlice(data.DefaultMaxBulkSize) mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ "h1": true, - }, buffSlice, "miniblocks") + }, buffSlice, "miniblocks", 1) expectedBuff = `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } { "doc" : { "receiverBlockHash" : "receiverBlock", "procTypeD": "Processed" } } diff --git a/process/elasticproc/operations/serialize_test.go b/process/elasticproc/operations/serialize_test.go index 63f2b204..d43aaabf 100644 --- a/process/elasticproc/operations/serialize_test.go +++ b/process/elasticproc/operations/serialize_test.go @@ -4,14 +4,13 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/stretchr/testify/require" ) func TestOperationsProcessor_SerializeSCRS(t *testing.T) { t.Parallel() - op, _ := NewOperationsProcessor(&mock.ShardCoordinatorMock{}) + op, _ := NewOperationsProcessor() scrs := []*data.ScResult{ { @@ -25,7 +24,7 @@ func TestOperationsProcessor_SerializeSCRS(t *testing.T) { } buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - err := op.SerializeSCRs(scrs, buffSlice, "operations") + err := op.SerializeSCRs(scrs, buffSlice, "operations", 0) require.Nil(t, err) require.Equal(t, `{"update":{"_index":"operations","_id":""}} {"script":{"source":"return"},"upsert":{"nonce":0,"gasLimit":0,"gasPrice":0,"value":"","sender":"","receiver":"","senderShard":0,"receiverShard":1,"prevTxHash":"","originalTxHash":"","callType":"","timestamp":0}} From e1121626829cb864b551fd67a4951099946dd2c3 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 21 Sep 2022 14:36:56 +0300 Subject: [PATCH 43/96] fix integration tests 1 --- .../accountsBalanceNftTransfer_test.go | 15 +++----- .../accountsBalanceWithLowerTimestamp_test.go | 20 +++++----- integrationtests/accountsESDTRollback_test.go | 15 ++------ .../accountsESDTWithTokenType_test.go | 38 +++++++------------ integrationtests/claimRewards_test.go | 10 ++--- integrationtests/collectionsIndex_test.go | 25 +++++------- integrationtests/createNFTWithTags_test.go | 16 +++----- integrationtests/esdtTransfer_test.go | 8 ++-- .../issueTokenAndSetRoles_test.go | 27 +++++-------- integrationtests/issueToken_test.go | 12 ++---- integrationtests/logsCrossShard_test.go | 11 ++---- integrationtests/nftIssueCreateBurn_test.go | 23 +++++------ .../nftTransferCrossShard_test.go | 26 +++++-------- .../nftTransferCrossWithScCall_test.go | 7 +--- integrationtests/relayedTx_test.go | 15 +++----- integrationtests/scCallIntraShard_test.go | 13 ++----- .../balance-nft-after-create.json | 2 +- .../balance-nft-after-transfer.json | 2 +- .../account-balance-esdt-first-update.json | 2 +- .../account-balance-esdt-second-update.json | 2 +- .../account-balance-first-update.json | 2 +- .../account-balance-second-update.json | 2 +- .../account-after-create.json | 2 +- .../account-esdt-with-type.json | 2 +- .../account-esdt-without-type.json | 2 +- .../account-esdt.json | 2 +- .../accounts-esdt-address-balance.json | 2 +- .../op-nft-transfer-sc-call-after-refund.json | 2 +- .../tx-complete-with-status.json | 2 +- .../tx-nft-transfer-failed-on-dst.json | 2 +- .../tx-nft-transfer-sc-call-after-refund.json | 2 +- .../tx-nft-transfer-sc-call-source.json | 2 +- .../tx-nft-transfer.json | 2 +- .../relayedTx/relayed-tx-after-refund.json | 2 +- .../testdata/relayedTx/relayed-tx-intra.json | 2 +- .../testdata/relayedTx/relayed-tx-source.json | 2 +- integrationtests/transactions_test.go | 7 +--- integrationtests/updateNFT_test.go | 16 ++++---- 38 files changed, 130 insertions(+), 214 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 3774ecc7..bf63121e 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -24,21 +23,18 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { require.Nil(t, err) // ################ CREATE NFT ########################## - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: 0, - } - body := &dataBlock.Body{} addr := "test-address-balance-1" addrHex := hex.EncodeToString([]byte(addr)) - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) header := &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 1, } pool := &outport.Pool{ @@ -72,7 +68,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids := []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} @@ -86,6 +82,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { header = &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 1, } pool = &outport.Pool{ @@ -109,7 +106,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { addrReceiver := "new-address" addrReceiverHex := hex.EncodeToString([]byte(addrReceiver)) - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) coreAlteredAccounts = map[string]*outport.AlteredAccount{ @@ -127,7 +124,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids = []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 2dd3ade3..5ac6fbbd 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -25,10 +24,6 @@ func TestIndexAccountsBalance(t *testing.T) { require.Nil(t, err) // ################ UPDATE ACCOUNT-ESDT BALANCE ########################## - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: 0, - } - body := &dataBlock.Body{} esdtToken := &esdt.ESDigitalToken{ @@ -57,12 +52,13 @@ func TestIndexAccountsBalance(t *testing.T) { encodedAddr2: alteredAccount, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) header := &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 2, } pool := &outport.Pool{ @@ -83,7 +79,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids := []string{"6161616162626262"} @@ -105,7 +101,7 @@ func TestIndexAccountsBalance(t *testing.T) { TimeStamp: 5000, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -124,6 +120,7 @@ func TestIndexAccountsBalance(t *testing.T) { header = &dataBlock.Header{ Round: 51, TimeStamp: 6000, + ShardID: 2, } coreAlteredAccounts[encodedAddr].Balance = "2000" @@ -159,7 +156,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -178,19 +175,20 @@ func TestIndexAccountsBalance(t *testing.T) { esdtToken.Value = big.NewInt(0) encodedAddr = hex.EncodeToString([]byte(addr)) - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) header = &dataBlock.Header{ Round: 51, TimeStamp: 6001, + ShardID: 2, } coreAlteredAccounts[encodedAddr].Balance = "2000" coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids = []string{"6161616162626262"} diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index df24063d..c41d3894 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -8,7 +8,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -25,13 +24,6 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: 1, - ComputeIdCalled: func(address []byte) uint32 { - return 1 - }, - } - esdtToken := &esdt.ESDigitalToken{ Value: big.NewInt(1000), Properties: []byte("ok"), @@ -57,7 +49,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { }, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) // CREATE SEMI-FUNGIBLE TOKEN @@ -84,9 +76,10 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: 2, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids := []string{"6161616162626262-TOKEN-eeee-02"} @@ -96,7 +89,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/accountsESDTRollback/account-after-create.json"), string(genericResponse.Docs[0].Source)) // DO ROLLBACK - err = esProc.RemoveAccountsESDT(5040) + err = esProc.RemoveAccountsESDT(5040, 2) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 683710b0..f9d52582 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -27,16 +26,13 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { require.Nil(t, err) // ################ ISSUE NON FUNGIBLE TOKEN ########################## - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) body := &dataBlock.Body{} header := &dataBlock.Header{ Round: 50, + ShardID: core.MetachainShardId, TimeStamp: 5040, } @@ -58,7 +54,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -68,10 +64,6 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/token-after-issue.json"), string(genericResponse.Docs[0].Source)) // ################ CREATE SEMI FUNGIBLE TOKEN ########################## - shardCoordinator = &mock.ShardCoordinatorMock{ - SelfID: 0, - } - addr := "aaaabbbb" encodedAddr := hex.EncodeToString([]byte(addr)) coreAlteredAccounts := map[string]*outport.AlteredAccount{ @@ -91,12 +83,13 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, }, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) header = &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 2, } esdtData := &esdt.ESDigitalToken{ @@ -124,7 +117,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids = []string{"6161616162626262-SEMI-abcd-02"} @@ -141,11 +134,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) esClient, err := createESClient(esURL) require.Nil(t, err) - // ################ CREATE SEMI FUNGIBLE TOKEN ########################## - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: 0, - } - + // ################ CREATE SEMI FUNGIBLE TOKEN ######################### body := &dataBlock.Body{} addr := "aaaabbbb" @@ -167,12 +156,13 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, }, } - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) header := &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 2, } esdtData := &esdt.ESDigitalToken{ @@ -200,7 +190,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids := []string{"6161616162626262-TTTT-abcd-02"} @@ -212,15 +202,13 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) time.Sleep(time.Second) // ################ ISSUE NON FUNGIBLE TOKEN ########################## - shardCoordinator = &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } header = &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: core.MetachainShardId, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) pool = &outport.Pool{ @@ -241,7 +229,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids = []string{"TTTT-abcd"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index 2b6edd32..243ce5bd 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -24,17 +23,14 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("claimRewards") header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: core.MetachainShardId, } scrHash1 := []byte("scrRefundGasReward") @@ -116,7 +112,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index 845b0bdd..e2b24518 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -8,7 +8,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -26,17 +25,14 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { require.Nil(t, err) // ################ ISSUE NON FUNGIBLE TOKEN ########################## - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) body := &dataBlock.Body{} header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: core.MetachainShardId, } pool := &outport.Pool{ @@ -57,14 +53,10 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) // ################ CREATE SEMI FUNGIBLE TOKEN 1 ########################## - shardCoordinator = &mock.ShardCoordinatorMock{ - SelfID: 0, - } - addr := "aaaabbbbcccccccc" addrHex := hex.EncodeToString([]byte(addr)) @@ -103,12 +95,13 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, }, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) header = &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 1, } esdtData := &esdt.ESDigitalToken{ @@ -136,7 +129,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids := []string{"61616161626262626363636363636363"} genericResponse := &GenericResponse{} @@ -166,7 +159,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 22 coreAlteredAccounts[addrForLogHex].Tokens[0].Nonce = 22 - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} @@ -194,7 +187,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) pool = &outport.Pool{ @@ -215,7 +208,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index bc11adc1..ffce5ce4 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -8,7 +8,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -34,19 +33,16 @@ func TestCreateNFTWithTags(t *testing.T) { }, } - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: 0, - } - header := &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 2, } addr := "aaaabbbb" addrHex := hex.EncodeToString([]byte(addr)) - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) esdtDataBytes, _ := json.Marshal(esdtToken) @@ -90,7 +86,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body := &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids := []string{"6161616162626262-DESK-abcd-01"} @@ -137,7 +133,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 2 body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) genericResponse = &GenericResponse{} @@ -163,7 +159,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 3 coreAlteredAccounts[addrHex].Tokens[0].MetaData.Attributes = attributes - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) pool = &outport.Pool{ @@ -185,7 +181,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) require.Nil(t, err) ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index e57928fe..382f6f7d 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -23,15 +22,14 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("esdtTransfer") header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: 0, } scrHash2 := []byte("scrHash2ESDTTransfer") body := &dataBlock.Body{ @@ -96,7 +94,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index fa7ebbd2..1e255134 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -6,7 +6,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -22,17 +21,14 @@ func TestIssueTokenAndSetRole(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) body := &dataBlock.Body{} header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: core.MetachainShardId, } pool := &outport.Pool{ @@ -58,7 +54,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -87,7 +83,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -120,7 +116,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -149,7 +145,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -165,17 +161,14 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) body := &dataBlock.Body{} header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: core.MetachainShardId, } // SET ROLES @@ -198,7 +191,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -226,7 +219,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index f2375299..c6d0d862 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -6,7 +6,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -22,17 +21,14 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) body := &dataBlock.Body{} header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: core.MetachainShardId, } pool := &outport.Pool{ @@ -53,7 +49,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{"SSSS-abcd"} @@ -82,7 +78,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids = []string{"SSSS-abcd"} diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 4b10f71e..9ea560ae 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -23,9 +22,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) header := &dataBlock.Header{ @@ -53,7 +50,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString([]byte("cross-log"))} @@ -94,7 +91,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) @@ -127,7 +124,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index da62bd4f..29b25971 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -24,18 +23,16 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - // ################ ISSUE NON FUNGIBLE TOKEN ########################## - shardCoordinator := &mock.ShardCoordinatorMock{ - SelfID: core.MetachainShardId, - } + // ################ ISSUE NON FUNGIBLE TOKEN ######################### - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) body := &dataBlock.Body{} header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: core.MetachainShardId, } pool := &outport.Pool{ @@ -56,7 +53,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{"NON-abcd"} @@ -66,16 +63,13 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/nftIssueCreateBurn/non-fungible-after-issue.json"), string(genericResponse.Docs[0].Source)) // ################ CREATE NON FUNGIBLE TOKEN ########################## - shardCoordinator = &mock.ShardCoordinatorMock{ - SelfID: 0, - } - - esProc, err = CreateElasticProcessor(esClient, shardCoordinator) + esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) header = &dataBlock.Header{ Round: 51, TimeStamp: 5600, + ShardID: 2, } esdtData := &esdt.ESDigitalToken{ @@ -103,7 +97,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids = []string{"NON-abcd-02"} @@ -117,6 +111,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { header = &dataBlock.Header{ Round: 52, TimeStamp: 5666, + ShardID: 2, } pool = &outport.Pool{ @@ -137,7 +132,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids = []string{"NON-abcd-02"} diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 1c72d2fe..df2fd2e6 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -23,14 +22,13 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("nftTransferWithSCCall") header := &dataBlock.Header{ Round: 50, + ShardID: 0, TimeStamp: 5040, } @@ -82,7 +80,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -122,7 +120,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -149,9 +147,7 @@ func TestNFTTransferCrossShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("nftTransfer") @@ -218,7 +214,7 @@ func TestNFTTransferCrossShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -256,7 +252,7 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -274,9 +270,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("nftTransferCross") @@ -324,7 +318,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -384,7 +378,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 3f28d78c..e06f3c2d 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -23,9 +22,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("nftTransferWithScCall") @@ -81,7 +78,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 7a751dd2..bba77d98 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -23,9 +22,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("relayedTx") @@ -84,7 +81,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -126,7 +123,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -144,9 +141,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("relayedTxIntra") @@ -217,7 +212,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index dce93e63..31067f65 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerData "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -23,9 +22,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("t") @@ -75,7 +72,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -95,9 +92,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("txHashClaimRewards") @@ -146,7 +141,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-create.json b/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-create.json index e83e56e3..d7c1d6eb 100644 --- a/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-create.json +++ b/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-create.json @@ -6,5 +6,5 @@ "tokenNonce": 7440483, "token": "NFT-abcdef", "timestamp": 5600, - "shardID": 0 + "shardID": 1 } diff --git a/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-transfer.json b/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-transfer.json index 0adfa4ab..5ff0c63e 100644 --- a/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-transfer.json +++ b/integrationtests/testdata/accountsBalanceNftTransfer/balance-nft-after-transfer.json @@ -6,5 +6,5 @@ "tokenNonce": 7440483, "token": "NFT-abcdef", "timestamp": 5600, - "shardID": 0 + "shardID": 1 } diff --git a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json index 1da62b33..31bafaf5 100644 --- a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json +++ b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json @@ -5,5 +5,5 @@ "token": "TTTT-abcd", "timestamp": 5600, "type": "FungibleESDT", - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-second-update.json b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-second-update.json index 469a1d04..70401f88 100644 --- a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-second-update.json +++ b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-second-update.json @@ -5,5 +5,5 @@ "timestamp": 6000, "token": "TTTT-abcd", "type": "FungibleESDT", - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json index 336d110e..96b6667e 100644 --- a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json +++ b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json @@ -4,5 +4,5 @@ "balanceNum": 0, "totalBalanceWithStake": "0", "timestamp": 5600, - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json index ebfe0be4..941a222a 100644 --- a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json +++ b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json @@ -4,5 +4,5 @@ "balanceNum": 0, "timestamp": 6000, "totalBalanceWithStake": "2000", - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/accountsESDTRollback/account-after-create.json b/integrationtests/testdata/accountsESDTRollback/account-after-create.json index 8a59d7d8..99c1574e 100644 --- a/integrationtests/testdata/accountsESDTRollback/account-after-create.json +++ b/integrationtests/testdata/accountsESDTRollback/account-after-create.json @@ -8,7 +8,7 @@ "whiteListedStorage": false, "nonEmptyURIs": false }, - "shardID": 1, + "shardID": 2, "tokenNonce": 2, "properties": "6f6b", "token": "TOKEN-eeee", diff --git a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json index 27b8fbdf..a9934b03 100644 --- a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json +++ b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json @@ -13,5 +13,5 @@ }, "timestamp": 5600, "type": "SemiFungibleESDT", - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json index ba3608b2..6d936d25 100644 --- a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json +++ b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json @@ -12,5 +12,5 @@ "whiteListedStorage": false }, "timestamp": 5600, - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json index 3f9e4418..abf69a78 100644 --- a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json +++ b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json @@ -14,5 +14,5 @@ "timestamp": 5600, "type": "SemiFungibleESDT", "currentOwner":"61646472", - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json b/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json index 7730e4f1..d113f36d 100644 --- a/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json +++ b/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json @@ -22,5 +22,5 @@ "properties": "6f6b", "token": "DESK-abcd", "timestamp": 5600, - "shardID": 0 + "shardID": 2 } diff --git a/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json b/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json index cb33ea19..c3fe92c4 100644 --- a/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json +++ b/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json @@ -28,7 +28,7 @@ "00000000000000000500f1c8f2fdc58a63c6b201fc2ed629962d3dfa33fe7ceb" ], "receiversShardIDs": [ - 0 + 1 ], "operation": "ESDTNFTTransfer", "type": "normal", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json index e06f49d3..d07f30aa 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json @@ -1,6 +1,6 @@ { "receiversShardIDs": [ - 0 + 2 ], "receiver": "65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376", "data": "RVNEVE5GVFRyYW5zZmVyQDQzNGY0YzRjNDU0MzU0NDk0ZjRlMmQzMjM2NjMzMTM4MzhAMDFAMDFAMDAwMDAwMDAwMDAwMDAwMDA1MDBhN2EwMjc3MWFhMDcwOTBlNjA3ZjAyYjI1ZjRkNmQyNDFiZmYzMmI5OTBhMg==", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json index a805e3c8..2c3bde08 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json @@ -28,7 +28,7 @@ "00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2" ], "receiversShardIDs": [ - 0 + 2 ], "operation": "ESDTNFTTransfer" } diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json index 2a886119..c052ff6a 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json @@ -28,7 +28,7 @@ "00000000000000000500f1c8f2fdc58a63c6b201fc2ed629962d3dfa33fe7ceb" ], "receiversShardIDs": [ - 0 + 1 ], "operation": "ESDTNFTTransfer", "function": "compoundRewardsProxy" diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json index 86cbb229..1ef472db 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json @@ -1,6 +1,6 @@ { "receiversShardIDs": [ - 0 + 1 ], "receiver": "657264316566397878336b336d3839617a6634633478633938777063646e78356830636e787936656d34377236646334616c756430757771783234663530", "data": "RVNEVE5GVFRyYW5zZmVyQDRjNGI0NjQxNTI0ZDJkMzM2NjM0NjYzOTYyQDAxNjUzNEA2ZjFlNmYwMWJjNzYyN2Y1YWVAMDAwMDAwMDAwMDAwMDAwMDA1MDBmMWM4ZjJmZGM1OGE2M2M2YjIwMWZjMmVkNjI5OTYyZDNkZmEzM2ZlN2NlYkA2MzZmNmQ3MDZmNzU2ZTY0NTI2NTc3NjE3MjY0NzM1MDcyNmY3ODc5QDAwMDAwMDAwMDAwMDAwMDAwNTAwNGY3OWVjNDRiYjEzMzcyYjVhYzlkOTk2ZDc0OTEyMGY0NzY0Mjc2MjdjZWI=", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json index ea21f90e..548f8804 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json @@ -1,6 +1,6 @@ { "receiversShardIDs": [ - 0 + 2 ], "receiver": "65726431757265376561323437636c6a3679716a673830756e7a36787a6a686c6a327a776d3467746736737564636d747364326377337873373468617376", "data": "RVNEVE5GVFRyYW5zZmVyQDUzNmY2ZDY1NzQ2ODY5NmU2NzJkNjE2MjYzNjQ2NTY2QDAxQDAxQDAwMDAwMDAwMDAwMDAwMDAwNTAwYTdhMDI3NzFhYTA3MDkwZTYwN2YwMmIyNWY0ZDZkMjQxYmZmMzJiOTkwYTI=", diff --git a/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json index 55f9dc69..774d9bcc 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json @@ -22,7 +22,7 @@ "000000000000000005008b60efce7fb25b140078645b71226cb9c644abc8a0d3" ], "receiversShardIDs": [ - 0 + 1 ], "operation": "transfer", "function": "saveAttestation", diff --git a/integrationtests/testdata/relayedTx/relayed-tx-intra.json b/integrationtests/testdata/relayedTx/relayed-tx-intra.json index 85f3219a..041fe85e 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-intra.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-intra.json @@ -22,7 +22,7 @@ "ae49d2246cf8ee248dc8a09dfcf3aaa6ec244f0844e349b31a35d94219dbfab3" ], "receiversShardIDs": [ - 0 + 1 ], "operation": "SaveKeyValue", "isRelayed": true diff --git a/integrationtests/testdata/relayedTx/relayed-tx-source.json b/integrationtests/testdata/relayedTx/relayed-tx-source.json index 0a367ae7..5ff3ab90 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-source.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-source.json @@ -22,7 +22,7 @@ "000000000000000005008b60efce7fb25b140078645b71226cb9c644abc8a0d3" ], "receiversShardIDs": [ - 0 + 1 ], "operation": "transfer", "function": "saveAttestation", diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index f19a11ae..0ba671d7 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerData "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" coreData "github.com/ElrondNetwork/elrond-go-core/data" dataBlock "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -22,9 +21,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { esClient, err := createESClient(esURL) require.Nil(t, err) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) txHash := []byte("hash") @@ -57,7 +54,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { string(txHash): tx, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index de7f02fb..8eb3d65b 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/mock" indexerdata "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -31,14 +30,13 @@ func TestNFTUpdateMetadata(t *testing.T) { } marshalizedCreate, _ := json.Marshal(esdtCreateData) - shardCoordinator := &mock.ShardCoordinatorMock{} - - esProc, err := CreateElasticProcessor(esClient, shardCoordinator) + esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) header := &dataBlock.Header{ Round: 50, TimeStamp: 5040, + ShardID: 2, } body := &dataBlock.Body{} @@ -60,7 +58,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids := []string{"NFT-abcd-0e"} @@ -87,7 +85,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) // Add URIS 2 --- results should be the same @@ -108,7 +106,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) // Update attributes 1 @@ -135,7 +133,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -163,7 +161,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false) + err = esProc.SaveTransactions(body, header, pool, nil, false, 3) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} From 2b74722fff9080f30619cdd6296560d4ee7159e2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 21 Sep 2022 16:16:24 +0300 Subject: [PATCH 44/96] fix integration tests 2 --- .../accountsBalanceWithLowerTimestamp_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 5ac6fbbd..45f75907 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -99,6 +99,7 @@ func TestIndexAccountsBalance(t *testing.T) { header = &dataBlock.Header{ Round: 51, TimeStamp: 5000, + ShardID: 2, } err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) @@ -124,11 +125,10 @@ func TestIndexAccountsBalance(t *testing.T) { } coreAlteredAccounts[encodedAddr].Balance = "2000" - pool = &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ "h1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: []byte("eeeebbbb"), + SndAddr: []byte(addr), }, 0, big.NewInt(0)), }, Logs: []*coreData.LogData{ @@ -150,8 +150,9 @@ func TestIndexAccountsBalance(t *testing.T) { body = &dataBlock.Body{ MiniBlocks: []*dataBlock.MiniBlock{ { - Type: dataBlock.TxBlock, - TxHashes: [][]byte{[]byte("h1")}, + Type: dataBlock.TxBlock, + TxHashes: [][]byte{[]byte("h1")}, + SenderShardID: 2, }, }, } From 4b904e13eee97283cd61c0629e73a82acfa47b1a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 22 Sep 2022 10:16:03 +0300 Subject: [PATCH 45/96] move factory --- {factory => process/factory}/indexerFactory.go | 0 {factory => process/factory}/indexerFactory_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {factory => process/factory}/indexerFactory.go (100%) rename {factory => process/factory}/indexerFactory_test.go (100%) diff --git a/factory/indexerFactory.go b/process/factory/indexerFactory.go similarity index 100% rename from factory/indexerFactory.go rename to process/factory/indexerFactory.go diff --git a/factory/indexerFactory_test.go b/process/factory/indexerFactory_test.go similarity index 100% rename from factory/indexerFactory_test.go rename to process/factory/indexerFactory_test.go From 3ab9c0d492a4352c30028ddc6d9daeb8b1ec3a3e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 22 Sep 2022 10:45:07 +0300 Subject: [PATCH 46/96] create ws client and indexer --- cmd/elasticindexer/config/config.toml | 2 + cmd/elasticindexer/main.go | 8 ++++ config/config.go | 7 ++- factory/wsIndexerFactory.go | 65 +++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 2 + process/wsclient/client.go | 12 +++-- process/wsindexer/indexer.go | 2 +- process/wsindexer/interface.go | 2 +- 9 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 factory/wsIndexerFactory.go diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index 77600346..f28b6273 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -21,6 +21,8 @@ type = "hex" [config.hasher] type = "blake2b" + [config.marshaller] + type = "gogo protobuf" [config.economics] denomination = 18 [config.logs] diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 3f70aeb9..9d79b9da 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ElrondNetwork/elastic-indexer-go/config" + "github.com/ElrondNetwork/elastic-indexer-go/factory" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" logger "github.com/ElrondNetwork/elrond-go-logger" @@ -70,6 +71,13 @@ func startIndexer(ctx *cli.Context) error { return err } + wsClient, err := factory.CreateWsIndexer(cfg) + if err != nil { + log.Error("cannot create ws indexer", "error", err) + } + + wsClient.Start() + log.Info("closing indexer") if !check.IfNil(fileLogging) { err = fileLogging.Close() diff --git a/config/config.go b/config/config.go index e635646b..5357093f 100644 --- a/config/config.go +++ b/config/config.go @@ -12,8 +12,8 @@ type Config struct { BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` WebSocket struct { - ServerURL string `toml:"server-url"` - DataMarshallerTyper string `toml:"data-marshaller-type"` + ServerURL string `toml:"server-url"` + DataMarshallerType string `toml:"data-marshaller-type"` } `toml:"web-socket"` AddressConverter struct { Length int `toml:"length"` @@ -26,6 +26,9 @@ type Config struct { Hasher struct { Type string `toml:"type"` } `toml:"hasher"` + Marshaller struct { + Type string `toml:"type"` + } `toml:"marshaller"` Economics struct { Denomination int `toml:"denomination"` } `toml:"economics"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go new file mode 100644 index 00000000..46c5046b --- /dev/null +++ b/factory/wsIndexerFactory.go @@ -0,0 +1,65 @@ +package factory + +import ( + "github.com/ElrondNetwork/elastic-indexer-go/config" + "github.com/ElrondNetwork/elastic-indexer-go/process/factory" + "github.com/ElrondNetwork/elastic-indexer-go/process/wsclient" + "github.com/ElrondNetwork/elastic-indexer-go/process/wsindexer" + "github.com/ElrondNetwork/elrond-go-core/core/pubkeyConverter" + factoryHasher "github.com/ElrondNetwork/elrond-go-core/hashing/factory" + factoryMarshaller "github.com/ElrondNetwork/elrond-go-core/marshal/factory" + logger "github.com/ElrondNetwork/elrond-go-logger" +) + +var log = logger.GetOrCreate("factory") + +func CreateWsIndexer(cfg *config.Config) (wsindexer.WSClient, error) { + dataIndexer, err := createDataIndexer(cfg) + if err != nil { + return nil, err + } + + wsMarshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.WebSocket.DataMarshallerType) + if err != nil { + return nil, err + } + + indexer, err := wsindexer.NewIndexer(wsMarshaller, dataIndexer) + + return wsclient.NewWebSocketClient(cfg.Config.WebSocket.ServerURL, indexer.GetFunctionsMap()) +} + +func createDataIndexer(cfg *config.Config) (wsindexer.DataIndexer, error) { + marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) + if err != nil { + return nil, err + } + hasher, err := factoryHasher.NewHasher(cfg.Config.Hasher.Type) + if err != nil { + return nil, err + } + addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, log) + if err != nil { + return nil, err + } + validatorPubkeyConverter, err := pubkeyConverter.NewHexPubkeyConverter(cfg.Config.ValidatorKeysConverter.Length) + if err != nil { + return nil, err + } + + return factory.NewIndexer(&factory.ArgsIndexerFactory{ + UseKibana: cfg.Config.ElasticCluster.UseKibana, + // Todo check if this is needed + IndexerCacheSize: 1, + Denomination: cfg.Config.Economics.Denomination, + BulkRequestMaxSize: cfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, + Url: cfg.Config.ElasticCluster.URL, + UserName: cfg.Config.ElasticCluster.UserName, + Password: cfg.Config.ElasticCluster.Password, + EnabledIndexes: cfg.Config.EnabledIndices, + Marshalizer: marshaller, + Hasher: hasher, + AddressPubkeyConverter: addressPubkeyConverter, + ValidatorPubkeyConverter: validatorPubkeyConverter, + }) +} diff --git a/go.mod b/go.mod index 2160762e..e44ef982 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd + github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374 github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 @@ -27,6 +27,7 @@ require ( github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect diff --git a/go.sum b/go.sum index 7a99b0e7..f7a19423 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd h1:3+wgMkVre4fEC3DW25NSEp8QK2gJsk1mGBH7nR9QYRk= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374 h1:6vw2Yz4qJrfJAuugPv7INg85VfpT7DuS/E+tuVv1Wj8= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= diff --git a/process/wsclient/client.go b/process/wsclient/client.go index 6ddfadcb..23535d4f 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -15,8 +15,6 @@ import ( "github.com/gorilla/websocket" ) -type Handler func(marshalledData []byte) error - type wsConn interface { io.Closer ReadMessage() (messageType int, p []byte, err error) @@ -29,10 +27,10 @@ var ( type client struct { urlReceive string - actions map[data.OperationType]Handler + actions map[data.OperationType]func(marshalledData []byte) error } -func NewWebSocketClient(urlReceive string, actions map[data.OperationType]Handler) (*client, error) { +func NewWebSocketClient(urlReceive string, actions map[data.OperationType]func(marshalledData []byte) error) (*client, error) { urlReceiveData := url.URL{Scheme: "ws", Host: fmt.Sprintf(urlReceive), Path: "/operations"} return &client{ @@ -139,7 +137,7 @@ func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsCo } } -// Stop - -func (c *client) Stop() { - // TODO implement +func (c *client) Close() { + //TODO implement me + panic("implement me") } diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 9acf0f8b..63b1d791 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -87,7 +87,7 @@ func (i *indexer) saveAccounts(marshalledData []byte) error { return err } - return i.di.SaveAccounts(argsSaveAccounts.BlockTimestamp, argsSaveAccounts.Acc) + return i.di.SaveAccounts(argsSaveAccounts.BlockTimestamp, argsSaveAccounts.Acc, argsSaveAccounts.ShardID) } func (i *indexer) finalizedBlock(_ []byte) error { diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index b8566811..bcca47cf 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -16,7 +16,7 @@ type DataIndexer interface { SaveRoundsInfo(roundsInfos []*outport.RoundInfo) error SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error SaveValidatorsRating(indexID string, infoRating []*outport.ValidatorRatingInfo) error - SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount) error + SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount, shardID uint32) error FinalizedBlock(headerHash []byte) error Close() error IsInterfaceNil() bool From 961c00b8bfce20a662e929bdd39db0a05e1cd9a0 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 22 Sep 2022 10:45:55 +0300 Subject: [PATCH 47/96] go mod tidy --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index f7a19423..afc92dcb 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd h1:3+wgMkVre4fEC3DW25NSEp8QK2gJsk1mGBH7nR9QYRk= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374 h1:6vw2Yz4qJrfJAuugPv7INg85VfpT7DuS/E+tuVv1Wj8= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= From f9655850e04ec5b0c8c4f0cad1586ac93d9b5caf Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 22 Sep 2022 17:22:50 +0300 Subject: [PATCH 48/96] refactor client in order to retry to connect --- process/wsclient/client.go | 59 ++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/process/wsclient/client.go b/process/wsclient/client.go index 23535d4f..b6dbb74d 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -22,7 +22,8 @@ type wsConn interface { } var ( - log = logger.GetOrCreate("process/wsclient") + log = logger.GetOrCreate("process/wsclient") + retryDuration = time.Second * 5 ) type client struct { @@ -45,28 +46,21 @@ func (c *client) Start() { log.Info("connecting to", "url", c.urlReceive) - wsConnection, _, err := websocket.DefaultDialer.Dial(c.urlReceive, nil) - if err != nil { - log.Error("dial", "error", err) - } - - defer func() { - err = wsConnection.Close() - log.LogIfError(err) - }() - + var wsConnection *websocket.Conn done := make(chan struct{}) go func() { defer close(done) for { - var message []byte - _, message, err = wsConnection.ReadMessage() + var err error + wsConnection, err = c.openConnection() if err != nil { - log.Error("error read message", "error", err) - return + log.Error(fmt.Sprintf("websocket error, retrying in %v...", retryDuration), "error", err.Error()) + time.Sleep(retryDuration) + continue } - c.verifyPayloadAndSendAckIfNeeded(message, wsConnection) + c.listeningOnWebSocket(wsConnection) + time.Sleep(retryDuration) } }() @@ -80,10 +74,13 @@ func (c *client) Start() { case <-timer.C: case <-interrupt: log.Info("interrupt") + if wsConnection == nil { + return + } // Cleanly close the connection by sending a close message and then // waiting (with timeout) for the server to close the connection. - err = wsConnection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + err := wsConnection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) if err != nil { log.Error("write close", "error", err) return @@ -97,6 +94,33 @@ func (c *client) Start() { } } +func (c *client) openConnection() (*websocket.Conn, error) { + conn, _, err := websocket.DefaultDialer.Dial(c.urlReceive, nil) + if err != nil { + return nil, err + } + + return conn, nil +} + +func (c *client) listeningOnWebSocket(wsConnection *websocket.Conn) { + for { + _, message, err := wsConnection.ReadMessage() + if err == nil { + c.verifyPayloadAndSendAckIfNeeded(message, wsConnection) + continue + } + + _, isConnectionClosed := err.(*websocket.CloseError) + if !isConnectionClosed { + log.Error("websocket error, retrying in %v...", "error", err.Error()) + } else { + log.Error(fmt.Sprintf("websocket terminated by the server side, retrying in %v...", retryDuration), "error", err.Error()) + } + return + } +} + func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsConn) { uint64ByteSliceConverter := uint64ByteSlice.NewBigEndianConverter() if len(payload) == 0 { @@ -115,7 +139,6 @@ func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsCo "counter", payloadData.Counter, "operation type", payloadData.OperationType, "message length", len(payloadData.Payload), - "data", payloadData.Payload, ) function, ok := c.actions[payloadData.OperationType] From ec81d2aed016fb3990c1c386322bfc7ab026706f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 27 Sep 2022 14:37:50 +0300 Subject: [PATCH 49/96] unmarshall data --- go.mod | 2 +- go.sum | 8 +- process/factory/indexerFactory.go | 4 - process/factory/indexerFactory_test.go | 12 -- process/wsindexer/indexer.go | 12 +- process/wsindexer/unmarsall.go | 208 +++++++++++++++++++++++++ process/wsindexer/unmarshall_test.go | 102 ++++++++++++ 7 files changed, 323 insertions(+), 25 deletions(-) create mode 100644 process/wsindexer/unmarsall.go create mode 100644 process/wsindexer/unmarshall_test.go diff --git a/go.mod b/go.mod index e44ef982..6ebe318d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374 + github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926115353-9b48b26bdf59 github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index afc92dcb..8f583893 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374 h1:6vw2Yz4qJrfJAuugPv7INg85VfpT7DuS/E+tuVv1Wj8= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220922073629-04049b54c374/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926074523-8cb6f5d0afd5 h1:ls9CPNQdrt6uGQy4ZJKHQf0O/znne6CWnqRsVQD0A7Q= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926074523-8cb6f5d0afd5/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926082128-a175b454ce36 h1:JMVeakddN50f/HubaaDskbADnXGJ0EjOhxww10T3Ris= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926082128-a175b454ce36/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926115353-9b48b26bdf59 h1:zU09YhIf2a5zg0ftOU+TWtcm1bUJWq5cH+nu7HSubvk= +github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926115353-9b48b26bdf59/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 5a58f8f6..cee9fb8f 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -47,10 +47,6 @@ func NewIndexer(args *ArgsIndexerFactory) (dataindexer.Indexer, error) { return nil, err } - if !args.Enabled { - return dataindexer.NewNilIndexer(), nil - } - elasticProcessor, err := createElasticProcessor(args) if err != nil { return nil, err diff --git a/process/factory/indexerFactory_test.go b/process/factory/indexerFactory_test.go index 4fb4f047..4ac70d85 100644 --- a/process/factory/indexerFactory_test.go +++ b/process/factory/indexerFactory_test.go @@ -106,18 +106,6 @@ func TestNewIndexerFactory(t *testing.T) { } } -func TestIndexerFactoryCreate_NilIndexer(t *testing.T) { - t.Parallel() - - args := createMockIndexerFactoryArgs() - args.Enabled = false - nilIndexer, err := NewIndexer(args) - require.NoError(t, err) - - _, ok := nilIndexer.(*dataindexer.NilIndexer) - require.True(t, ok) -} - func TestIndexerFactoryCreate_ElasticIndexer(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) args := createMockIndexerFactoryArgs() diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 63b1d791..04fdc3a1 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -1,11 +1,13 @@ package wsindexer import ( - "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" + logger "github.com/ElrondNetwork/elrond-go-logger" ) +var log = logger.GetOrCreate("process/wsindexer") + type indexer struct { marshaller marshal.Marshalizer di DataIndexer @@ -31,8 +33,7 @@ func (i *indexer) GetFunctionsMap() map[data.OperationType]func(d []byte) error } func (i *indexer) saveBlock(marshalledData []byte) error { - argsSaveBlock := &outport.ArgsSaveBlockData{} - err := i.marshaller.Unmarshal(argsSaveBlock, marshalledData) + argsSaveBlock, err := i.getArgsSaveBlock(marshalledData) if err != nil { return err } @@ -41,13 +42,12 @@ func (i *indexer) saveBlock(marshalledData []byte) error { } func (i *indexer) revertIndexedBlock(marshalledData []byte) error { - argsRevert := &data.ArgsRevertIndexedBlock{} - err := i.marshaller.Unmarshal(argsRevert, marshalledData) + header, body, err := i.getHeaderAndBody(marshalledData) if err != nil { return err } - return i.di.RevertIndexedBlock(argsRevert.Header, argsRevert.Body) + return i.di.RevertIndexedBlock(header, body) } func (i *indexer) saveRounds(marshalledData []byte) error { diff --git a/process/wsindexer/unmarsall.go b/process/wsindexer/unmarsall.go new file mode 100644 index 00000000..c1d0ad23 --- /dev/null +++ b/process/wsindexer/unmarsall.go @@ -0,0 +1,208 @@ +package wsindexer + +import ( + "errors" + "time" + + "github.com/ElrondNetwork/elrond-go-core/data" + "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" + "github.com/ElrondNetwork/elrond-go-core/data/receipt" + "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" + "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" + "github.com/ElrondNetwork/elrond-go-core/data/transaction" + outportData "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" +) + +func (i *indexer) getArgsSaveBlock(marshaledData []byte) (*outport.ArgsSaveBlockData, error) { + defer func(start time.Time) { + log.Debug("indexer.getArgsSaveBlock", "duration", time.Since(start)) + }(time.Now()) + + header, err := i.getHeader(marshaledData) + if err != nil { + return nil, err + } + + body, err := i.getBody(marshaledData) + if err != nil { + return nil, err + } + + txsPool, err := i.getTxsPool(marshaledData) + if err != nil { + return nil, err + } + + argsBlockS := struct { + HeaderHash []byte + SignersIndexes []uint64 + NotarizedHeadersHashes []string + HeaderGasConsumption outport.HeaderGasConsumption + AlteredAccounts map[string]*outport.AlteredAccount + NumberOfShards uint32 + IsImportDB bool + }{} + err = i.marshaller.Unmarshal(&argsBlockS, marshaledData) + if err != nil { + return nil, err + } + + return &outport.ArgsSaveBlockData{ + HeaderHash: argsBlockS.HeaderHash, + Body: body, + Header: header, + SignersIndexes: argsBlockS.SignersIndexes, + NotarizedHeadersHashes: argsBlockS.NotarizedHeadersHashes, + HeaderGasConsumption: argsBlockS.HeaderGasConsumption, + TransactionsPool: txsPool, + AlteredAccounts: argsBlockS.AlteredAccounts, + NumberOfShards: argsBlockS.NumberOfShards, + IsImportDB: argsBlockS.IsImportDB, + }, nil +} + +func (i *indexer) getTxsPool(marshaledData []byte) (*outport.Pool, error) { + type normalTxWrapped struct { + TransactionHandler *transaction.Transaction + outport.FeeInfo + } + type rewardsTxsWrapped struct { + TransactionHandler *rewardTx.RewardTx + outport.FeeInfo + } + type scrWrapped struct { + TransactionHandler *smartContractResult.SmartContractResult + outport.FeeInfo + } + type receiptWrapped struct { + TransactionHandler *receipt.Receipt + outport.FeeInfo + } + type logWrapped struct { + TxHash string + transaction.Log + } + + type poolStruct struct { + Txs map[string]*normalTxWrapped + Invalid map[string]*normalTxWrapped + Scrs map[string]*scrWrapped + Rewards map[string]*rewardsTxsWrapped + Receipts map[string]*receiptWrapped + Logs []*logWrapped + } + + argSaveBlock := struct { + TransactionsPool *poolStruct + }{} + + err := i.marshaller.Unmarshal(&argSaveBlock, marshaledData) + if err != nil { + return nil, err + } + + normalTxs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Txs)) + for txHash, tx := range argSaveBlock.TransactionsPool.Txs { + normalTxs[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + normalTxs[txHash].SetInitialPaidFee(tx.InitialPaidFee) + } + + invalidTxs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Invalid)) + for txHash, tx := range argSaveBlock.TransactionsPool.Invalid { + invalidTxs[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + invalidTxs[txHash].SetInitialPaidFee(tx.InitialPaidFee) + } + + scrs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Scrs)) + for txHash, tx := range argSaveBlock.TransactionsPool.Scrs { + scrs[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + scrs[txHash].SetInitialPaidFee(tx.InitialPaidFee) + } + + receipts := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Receipts)) + for txHash, tx := range argSaveBlock.TransactionsPool.Receipts { + receipts[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + receipts[txHash].SetInitialPaidFee(tx.InitialPaidFee) + } + + rewards := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Rewards)) + for txHash, tx := range argSaveBlock.TransactionsPool.Rewards { + rewards[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + rewards[txHash].SetInitialPaidFee(tx.InitialPaidFee) + } + + logs := make([]*data.LogData, 0, len(argSaveBlock.TransactionsPool.Logs)) + for _, txLog := range argSaveBlock.TransactionsPool.Logs { + logs = append(logs, &data.LogData{ + LogHandler: txLog, + TxHash: txLog.TxHash, + }) + } + + return &outport.Pool{ + Txs: normalTxs, + Scrs: scrs, + Rewards: rewards, + Invalid: invalidTxs, + Receipts: receipts, + Logs: logs, + }, nil +} + +func (i *indexer) getHeaderAndBody(marshaledData []byte) (data.HeaderHandler, data.BodyHandler, error) { + body, err := i.getBody(marshaledData) + if err != nil { + return nil, nil, err + } + + header, err := i.getHeader(marshaledData) + if err != nil { + return nil, nil, err + } + + return header, body, nil +} + +func (i *indexer) getBody(marshaledData []byte) (data.BodyHandler, error) { + bodyStruct := struct { + Body *block.Body + }{} + + err := i.marshaller.Unmarshal(&bodyStruct, marshaledData) + return bodyStruct.Body, err +} + +func (i *indexer) getHeader(marshaledData []byte) (data.HeaderHandler, error) { + headerTypeStruct := struct { + HeaderType outportData.HeaderType + }{} + + err := i.marshaller.Unmarshal(&headerTypeStruct, marshaledData) + if err != nil { + return nil, err + } + + switch headerTypeStruct.HeaderType { + case outportData.MetaHeader: + hStruct := struct { + H1 *block.Header `json:"Header"` + }{} + err = i.marshaller.Unmarshal(&hStruct, marshaledData) + return hStruct.H1, err + case outportData.ShardHeaderV1: + hStruct := struct { + H1 *block.MetaBlock `json:"Header"` + }{} + err = i.marshaller.Unmarshal(&hStruct, marshaledData) + return hStruct.H1, err + case outportData.ShardHeaderV2: + hStruct := struct { + H1 *block.HeaderV2 `json:"Header"` + }{} + err = i.marshaller.Unmarshal(&hStruct, marshaledData) + return hStruct.H1, err + default: + return nil, errors.New("invalid header type") + } +} diff --git a/process/wsindexer/unmarshall_test.go b/process/wsindexer/unmarshall_test.go new file mode 100644 index 00000000..1f7518ad --- /dev/null +++ b/process/wsindexer/unmarshall_test.go @@ -0,0 +1,102 @@ +package wsindexer + +import ( + "math/big" + "testing" + + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + coreData "github.com/ElrondNetwork/elrond-go-core/data" + "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go-core/data/outport" + "github.com/ElrondNetwork/elrond-go-core/data/receipt" + "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" + "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" + "github.com/ElrondNetwork/elrond-go-core/data/transaction" + "github.com/ElrondNetwork/elrond-go-core/marshal/factory" + outportData "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" + "github.com/stretchr/testify/require" +) + +func TestGetHeaderAndBody(t *testing.T) { + t.Parallel() + + marshaller, _ := factory.NewMarshalizer("json") + nilDataIndexer := dataindexer.NewNilIndexer() + + di, _ := NewIndexer(marshaller, nilDataIndexer) + + arg := &outportData.ArgsRevertIndexedBlock{ + HeaderType: outportData.ShardHeaderV2, + Header: &block.HeaderV2{ScheduledRootHash: []byte("aaaaaa")}, + Body: &block.Body{MiniBlocks: []*block.MiniBlock{{}}}, + } + argBytes, _ := marshaller.Marshal(arg) + + body, header, err := di.getHeaderAndBody(argBytes) + require.Nil(t, err) + require.NotNil(t, body) + require.NotNil(t, header) +} + +func TestGetPool(t *testing.T) { + t.Parallel() + + marshaller, _ := factory.NewMarshalizer("json") + nilDataIndexer := dataindexer.NewNilIndexer() + + di, _ := NewIndexer(marshaller, nilDataIndexer) + + pool := &outport.Pool{ + Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "txHash": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 1, + }, 1, big.NewInt(100)), + }, + Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "scrHash": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + Nonce: 2, + }, 0, big.NewInt(0)), + }, + Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "reward": outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ + Value: big.NewInt(10), + }, 0, big.NewInt(0)), + }, + Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "invalid": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + Nonce: 3, + }, 100, big.NewInt(1000)), + }, + Receipts: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + "rec": outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + Value: big.NewInt(300), + }, 0, big.NewInt(0)), + }, + Logs: []*coreData.LogData{ + { + TxHash: "something", + LogHandler: &transaction.Log{ + Address: []byte("addr"), + }, + }, + }, + } + + argsSaveBlock := &outportData.ArgsSaveBlock{ + ArgsSaveBlockData: &outport.ArgsSaveBlockData{ + TransactionsPool: pool, + }, + } + + argsSaveBlockBytes, _ := di.marshaller.Marshal(argsSaveBlock) + + resPool, err := di.getTxsPool(argsSaveBlockBytes) + require.Nil(t, err) + require.NotNil(t, resPool) + require.Equal(t, 1, len(resPool.Txs)) + require.Equal(t, 1, len(resPool.Scrs)) + require.Equal(t, 1, len(resPool.Rewards)) + require.Equal(t, 1, len(resPool.Receipts)) + require.Equal(t, 1, len(resPool.Invalid)) + require.Equal(t, 1, len(resPool.Logs)) +} From 20f3e2311e547aae4547203636a510ce85708850 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 27 Sep 2022 14:39:30 +0300 Subject: [PATCH 50/96] latest elrond go core --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 6ebe318d..fedb56cf 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926115353-9b48b26bdf59 + github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220927113554-34ccab6be8c3 github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 8f583893..af25bf3b 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926074523-8cb6f5d0afd5 h1:ls9CPNQdrt6uGQy4ZJKHQf0O/znne6CWnqRsVQD0A7Q= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926074523-8cb6f5d0afd5/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926082128-a175b454ce36 h1:JMVeakddN50f/HubaaDskbADnXGJ0EjOhxww10T3Ris= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926082128-a175b454ce36/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926115353-9b48b26bdf59 h1:zU09YhIf2a5zg0ftOU+TWtcm1bUJWq5cH+nu7HSubvk= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220926115353-9b48b26bdf59/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220927113554-34ccab6be8c3 h1:20nr2EMpS10pYr2Te/DXwIiAH8C86YvtTMB4kxSfw9g= +github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220927113554-34ccab6be8c3/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= From bab1d996901b4b90754732cf8ff1a2fd73027772 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 28 Sep 2022 14:14:15 +0300 Subject: [PATCH 51/96] fix unmarshal --- process/wsindexer/unmarsall.go | 48 ++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/process/wsindexer/unmarsall.go b/process/wsindexer/unmarsall.go index c1d0ad23..3a420058 100644 --- a/process/wsindexer/unmarsall.go +++ b/process/wsindexer/unmarsall.go @@ -1,6 +1,7 @@ package wsindexer import ( + "encoding/hex" "errors" "time" @@ -80,8 +81,8 @@ func (i *indexer) getTxsPool(marshaledData []byte) (*outport.Pool, error) { outport.FeeInfo } type logWrapped struct { - TxHash string - transaction.Log + TxHash string + LogHandler *transaction.Log } type poolStruct struct { @@ -104,39 +105,49 @@ func (i *indexer) getTxsPool(marshaledData []byte) (*outport.Pool, error) { normalTxs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Txs)) for txHash, tx := range argSaveBlock.TransactionsPool.Txs { - normalTxs[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - normalTxs[txHash].SetInitialPaidFee(tx.InitialPaidFee) + decoded := getDecodedHash(txHash) + + normalTxs[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + normalTxs[decoded].SetInitialPaidFee(tx.InitialPaidFee) } invalidTxs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Invalid)) for txHash, tx := range argSaveBlock.TransactionsPool.Invalid { - invalidTxs[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - invalidTxs[txHash].SetInitialPaidFee(tx.InitialPaidFee) + decoded := getDecodedHash(txHash) + + invalidTxs[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + invalidTxs[decoded].SetInitialPaidFee(tx.InitialPaidFee) } scrs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Scrs)) for txHash, tx := range argSaveBlock.TransactionsPool.Scrs { - scrs[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - scrs[txHash].SetInitialPaidFee(tx.InitialPaidFee) + decoded := getDecodedHash(txHash) + + scrs[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + scrs[decoded].SetInitialPaidFee(tx.InitialPaidFee) } receipts := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Receipts)) for txHash, tx := range argSaveBlock.TransactionsPool.Receipts { - receipts[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - receipts[txHash].SetInitialPaidFee(tx.InitialPaidFee) + decoded := getDecodedHash(txHash) + + receipts[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + receipts[decoded].SetInitialPaidFee(tx.InitialPaidFee) } rewards := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Rewards)) for txHash, tx := range argSaveBlock.TransactionsPool.Rewards { - rewards[txHash] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - rewards[txHash].SetInitialPaidFee(tx.InitialPaidFee) + decoded := getDecodedHash(txHash) + + rewards[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) + rewards[decoded].SetInitialPaidFee(tx.InitialPaidFee) } logs := make([]*data.LogData, 0, len(argSaveBlock.TransactionsPool.Logs)) for _, txLog := range argSaveBlock.TransactionsPool.Logs { logs = append(logs, &data.LogData{ - LogHandler: txLog, - TxHash: txLog.TxHash, + LogHandler: txLog.LogHandler, + TxHash: getDecodedHash(txLog.TxHash), }) } @@ -150,6 +161,15 @@ func (i *indexer) getTxsPool(marshaledData []byte) (*outport.Pool, error) { }, nil } +func getDecodedHash(hash string) string { + decoded, err := hex.DecodeString(hash) + if err != nil { + log.Warn("getDecodedHash.cannot decode hash", "error", err, "hash", hash) + return hash + } + return string(decoded) +} + func (i *indexer) getHeaderAndBody(marshaledData []byte) (data.HeaderHandler, data.BodyHandler, error) { body, err := i.getBody(marshaledData) if err != nil { From a435a28606ccbd460940fdd5ce5f0aef1c371253 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 28 Sep 2022 14:27:13 +0300 Subject: [PATCH 52/96] remove comment code and extra checks --- .../logsevents/delegatorsProcessor.go | 4 +++ .../logsevents/delegatorsProcessor_test.go | 29 +++++++++++-------- .../logsevents/esdtIssueProcessor.go | 4 +++ .../logsevents/esdtIssueProcessor_test.go | 10 ++++--- .../logsevents/logsAndEventsProcessor.go | 12 +++----- .../logsevents/logsAndEventsProcessor_test.go | 2 +- 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/process/elasticproc/logsevents/delegatorsProcessor.go b/process/elasticproc/logsevents/delegatorsProcessor.go index f8618042..61d4d3f3 100644 --- a/process/elasticproc/logsevents/delegatorsProcessor.go +++ b/process/elasticproc/logsevents/delegatorsProcessor.go @@ -42,6 +42,10 @@ func newDelegatorsProcessor( } func (dp *delegatorsProc) processEvent(args *argsProcessEvent) argOutputProcessEvent { + if args.selfShardID != core.MetachainShardId { + return argOutputProcessEvent{} + } + eventIdentifierStr := string(args.event.GetIdentifier()) _, ok := dp.delegatorsOperations[eventIdentifierStr] if !ok { diff --git a/process/elasticproc/logsevents/delegatorsProcessor_test.go b/process/elasticproc/logsevents/delegatorsProcessor_test.go index 4d50571a..7d9b6fa0 100644 --- a/process/elasticproc/logsevents/delegatorsProcessor_test.go +++ b/process/elasticproc/logsevents/delegatorsProcessor_test.go @@ -8,6 +8,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/stretchr/testify/require" ) @@ -21,9 +22,10 @@ func TestDelegatorsProcessor_ProcessEvent(t *testing.T) { Topics: [][]byte{big.NewInt(1000).Bytes(), big.NewInt(1000000000).Bytes(), big.NewInt(10).Bytes(), big.NewInt(1000000000).Bytes()}, } args := &argsProcessEvent{ - timestamp: 1234, - event: event, - logAddress: []byte("contract"), + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + selfShardID: core.MetachainShardId, } balanceConverter, _ := converters.NewBalanceConverter(10) @@ -48,9 +50,10 @@ func TestDelegatorProcessor_WithdrawWithDelete(t *testing.T) { Topics: [][]byte{big.NewInt(1000).Bytes(), big.NewInt(0).Bytes(), big.NewInt(10).Bytes(), big.NewInt(1000000000).Bytes(), []byte(strconv.FormatBool(true))}, } args := &argsProcessEvent{ - timestamp: 1234, - event: event, - logAddress: []byte("contract"), + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + selfShardID: core.MetachainShardId, } balanceConverter, _ := converters.NewBalanceConverter(10) @@ -76,9 +79,10 @@ func TestDelegatorProcessor_ClaimRewardsWithDelete(t *testing.T) { Topics: [][]byte{big.NewInt(1000).Bytes(), []byte(strconv.FormatBool(true))}, } args := &argsProcessEvent{ - timestamp: 1234, - event: event, - logAddress: []byte("contract"), + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + selfShardID: core.MetachainShardId, } balanceConverter, _ := converters.NewBalanceConverter(10) @@ -102,9 +106,10 @@ func TestDelegatorProcessor_ClaimRewardsNoDelete(t *testing.T) { Topics: [][]byte{big.NewInt(1000).Bytes(), []byte(strconv.FormatBool(false))}, } args := &argsProcessEvent{ - timestamp: 1234, - event: event, - logAddress: []byte("contract"), + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + selfShardID: core.MetachainShardId, } balanceConverter, _ := converters.NewBalanceConverter(10) diff --git a/process/elasticproc/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go index a51d8379..b127d2e5 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor.go @@ -41,6 +41,10 @@ func newESDTIssueProcessor(pubkeyConverter core.PubkeyConverter) *esdtIssueProce } func (iep *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { + if args.selfShardID != core.MetachainShardId { + return argOutputProcessEvent{} + } + identifierStr := string(args.event.GetIdentifier()) _, ok := iep.issueOperationsIdentifiers[identifierStr] if !ok { diff --git a/process/elasticproc/logsevents/esdtIssueProcessor_test.go b/process/elasticproc/logsevents/esdtIssueProcessor_test.go index 914a7462..34da0685 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor_test.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor_test.go @@ -22,8 +22,9 @@ func TestIssueESDTProcessor(t *testing.T) { Topics: [][]byte{[]byte("MYTOKEN-abcd"), []byte("my-token"), []byte("MYTOKEN"), []byte(core.NonFungibleESDT)}, } args := &argsProcessEvent{ - timestamp: 1234, - event: event, + timestamp: 1234, + event: event, + selfShardID: core.MetachainShardId, } res := esdtIssueProc.processEvent(args) @@ -57,8 +58,9 @@ func TestIssueESDTProcessor_TransferOwnership(t *testing.T) { Topics: [][]byte{[]byte("MYTOKEN-abcd"), []byte("my-token"), []byte("MYTOKEN"), []byte(core.NonFungibleESDT), []byte("newOwner")}, } args := &argsProcessEvent{ - timestamp: 1234, - event: event, + timestamp: 1234, + event: event, + selfShardID: core.MetachainShardId, } res := esdtIssueProc.processEvent(args) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index ab4d36f3..2526f58f 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -69,6 +69,8 @@ func createEventsProcessors(args *ArgsLogsAndEventsProcessor) []eventsProcessor informativeProc := newInformativeLogsProcessor() updateNFTProc := newNFTsPropertiesProcessor(args.PubKeyConverter) esdtPropProc := newEsdtPropertiesProcessor(args.PubKeyConverter) + esdtIssueProc := newESDTIssueProcessor(args.PubKeyConverter) + delegatorsProcessor := newDelegatorsProcessor(args.PubKeyConverter, args.BalanceConverter) eventsProcs := []eventsProcessor{ fungibleProc, @@ -77,16 +79,10 @@ func createEventsProcessors(args *ArgsLogsAndEventsProcessor) []eventsProcessor informativeProc, updateNFTProc, esdtPropProc, + esdtIssueProc, + delegatorsProcessor, } - //if args.ShardCoordinator.SelfId() == core.MetachainShardId { - esdtIssueProc := newESDTIssueProcessor(args.PubKeyConverter) - eventsProcs = append(eventsProcs, esdtIssueProc) - - delegatorsProcessor := newDelegatorsProcessor(args.PubKeyConverter, args.BalanceConverter) - eventsProcs = append(eventsProcs, delegatorsProcessor) - //} - return eventsProcs } diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 575e0393..8c5963ec 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -150,7 +150,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) args.BalanceConverter = balanceConverter proc, _ := NewLogsAndEventsProcessor(args) - resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000, 0, 3) + resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000, core.MetachainShardId, 3) require.NotNil(t, resLogs.Tokens) require.True(t, res.Transactions[0].HasOperations) require.True(t, res.ScResults[0].HasOperations) From 7cebfc3187ce7331e6ce33f155a08937367a7fef Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 28 Sep 2022 15:28:26 +0300 Subject: [PATCH 53/96] change log --- process/wsclient/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process/wsclient/client.go b/process/wsclient/client.go index b6dbb74d..f9d4ff80 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -113,9 +113,9 @@ func (c *client) listeningOnWebSocket(wsConnection *websocket.Conn) { _, isConnectionClosed := err.(*websocket.CloseError) if !isConnectionClosed { - log.Error("websocket error, retrying in %v...", "error", err.Error()) + log.Warn("websocket error, retrying in %v...", "error", err.Error()) } else { - log.Error(fmt.Sprintf("websocket terminated by the server side, retrying in %v...", retryDuration), "error", err.Error()) + log.Warn(fmt.Sprintf("websocket terminated by the server side, retrying in %v...", retryDuration), "error", err.Error()) } return } From 2d410cf8a28cf18a2c7301e4280cdc9436732258 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 28 Sep 2022 15:29:00 +0300 Subject: [PATCH 54/96] change log --- process/wsclient/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/wsclient/client.go b/process/wsclient/client.go index f9d4ff80..35d6fd5c 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -54,7 +54,7 @@ func (c *client) Start() { var err error wsConnection, err = c.openConnection() if err != nil { - log.Error(fmt.Sprintf("websocket error, retrying in %v...", retryDuration), "error", err.Error()) + log.Warn(fmt.Sprintf("websocket error, retrying in %v...", retryDuration), "error", err.Error()) time.Sleep(retryDuration) continue } From b5d88710e7ffb7b5b3b31d4069690860ef73a988 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 28 Sep 2022 16:05:51 +0300 Subject: [PATCH 55/96] ignore peer blocks --- process/elasticproc/block/blockProcessor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/process/elasticproc/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go index 6f3206df..4dec2c2a 100644 --- a/process/elasticproc/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -128,6 +128,10 @@ func getTxsCount(header coreData.HeaderHandler) (numTxs, notarizedTxs uint32) { notarizedTxs = metaHeader.TxCount numTxs = 0 for _, mb := range metaHeader.MiniBlockHeaders { + if mb.Type == block.PeerBlock { + continue + } + numTxs += mb.TxCount } From 6de5af7e1755e6c244a2b9fdb32688dafc10bf34 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 30 Sep 2022 11:32:49 +0300 Subject: [PATCH 56/96] fixes after review 1 --- .github/workflows/golangci-lint.yml | 37 ++++++++++++++ cmd/elasticindexer/config/config.toml | 6 --- cmd/elasticindexer/config/prefs.toml | 6 +++ cmd/elasticindexer/flags.go | 7 +++ cmd/elasticindexer/main.go | 42 +++++++++++----- config/config.go | 20 ++++---- factory/wsIndexerFactory.go | 31 +++++++----- .../accountsBalanceNftTransfer_test.go | 4 +- .../accountsBalanceWithLowerTimestamp_test.go | 8 ++-- integrationtests/accountsESDTRollback_test.go | 2 +- .../accountsESDTWithTokenType_test.go | 8 ++-- integrationtests/claimRewards_test.go | 2 +- integrationtests/collectionsIndex_test.go | 8 ++-- integrationtests/consts.go | 6 +++ integrationtests/createNFTWithTags_test.go | 6 +-- integrationtests/esdtTransfer_test.go | 2 +- .../issueTokenAndSetRoles_test.go | 12 ++--- integrationtests/issueToken_test.go | 4 +- integrationtests/logsCrossShard_test.go | 6 +-- integrationtests/nftIssueCreateBurn_test.go | 6 +-- .../nftTransferCrossShard_test.go | 12 ++--- .../nftTransferCrossWithScCall_test.go | 2 +- integrationtests/relayedTx_test.go | 6 +-- integrationtests/scCallIntraShard_test.go | 4 +- integrationtests/transactions_test.go | 2 +- integrationtests/updateNFT_test.go | 10 ++-- integrationtests/utils.go | 2 - process/dataindexer/dataDispatcher_test.go | 20 ++++---- process/dataindexer/errors.go | 3 -- process/elasticproc/elasticProcessor_test.go | 2 +- .../factory/elasticProcessorFactory.go | 2 +- .../logsevents/esdtIssueProcessor.go | 8 ++-- .../logsevents/esdtIssueProcessor_test.go | 20 ++++++++ .../logsevents/logsAndEventsProcessor.go | 6 +-- .../logsevents/logsAndEventsProcessor_test.go | 4 +- .../elasticproc/logsevents/nftsProcessor.go | 10 ++-- process/elasticproc/logsevents/serialize.go | 48 +++++++++---------- process/elasticproc/tags/serialize.go | 6 +-- process/elasticproc/transactions/serialize.go | 18 +++---- process/factory/indexerFactory.go | 11 ++--- process/factory/indexerFactory_test.go | 20 ++++---- process/wsclient/client.go | 22 +++++---- process/wsindexer/indexer.go | 17 ++++++- process/wsindexer/interface.go | 2 + .../wsindexer/{unmarsall.go => unmarshal.go} | 0 .../{unmarshall_test.go => unmarshal_test.go} | 0 46 files changed, 299 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 cmd/elasticindexer/config/prefs.toml create mode 100644 integrationtests/consts.go rename process/wsindexer/{unmarsall.go => unmarshal.go} (100%) rename process/wsindexer/{unmarshall_test.go => unmarshal_test.go} (100%) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 00000000..c63a71a9 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,37 @@ +name: golangci-lint +on: + push: + branches: + - master + pull_request: + branches: [ master, development, feat/*, rc/* ] + +permissions: + contents: read + +jobs: + golangci: + name: golangci linter + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: 1.17.6 + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. + version: v1.45.2 + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + args: --timeout 10m0s --max-issues-per-linter 0 --max-same-issues 0 --print-issued-lines + + # Optional: show only new issues if it's a pull request. The default value is `false`. + only-new-issues: true + + # Optional: if set to true then the action will use pre-installed Go + # skip-go-installation: true diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index f28b6273..3a58f705 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -4,12 +4,6 @@ "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "collections" ] - [config.elastic-cluster] - use-kibana = false - url = "http://localhost:9200" - username = "" - password = "" - bulk-request-max-size-in-bytes = 4194304 # 4MB [config.web-socket] server-url = "localhost:22111" data-marshaller-type = "json" diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml new file mode 100644 index 00000000..e134fd36 --- /dev/null +++ b/cmd/elasticindexer/config/prefs.toml @@ -0,0 +1,6 @@ +[elastic-cluster] + use-kibana = false + url = "http://localhost:9200" + username = "" + password = "" + bulk-request-max-size-in-bytes = 4194304 # 4MB diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index 5ec81789..489d6550 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -11,6 +11,13 @@ var ( Usage: "The main configuration file to load", Value: "./config/config.toml", } + // configurationPreferencesFile defines a flag for the path to the preferences toml configuration file + configurationPreferencesFile = cli.StringFlag{ + Name: "config-preferences", + Usage: "The [path] for the preferences configuration file. This TOML file contains " + + "preferences configurations, such as the node display name or the shard to start in when starting as observer", + Value: "./config/prefs.toml", + } logLevel = cli.StringFlag{ Name: "log-level", Usage: "This flag specifies the logger `level(s)`. It can contain multiple comma-separated value. For example" + diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 9d79b9da..da598221 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -40,6 +40,7 @@ func main() { app.Usage = "This tool will index data in an Elasticsearch database" app.Flags = []cli.Flag{ configurationFile, + configurationPreferencesFile, logLevel, logSaveFile, } @@ -60,8 +61,12 @@ func main() { } func startIndexer(ctx *cli.Context) error { - configurationFileName := ctx.GlobalString(configurationFile.Name) - cfg, err := loadMainConfig(configurationFileName) + cfg, err := loadMainConfig(ctx.GlobalString(configurationFile.Name)) + if err != nil { + return err + } + + clusterCfg, err := loadClusterConfig(ctx.GlobalString(configurationPreferencesFile.Name)) if err != nil { return err } @@ -71,7 +76,7 @@ func startIndexer(ctx *cli.Context) error { return err } - wsClient, err := factory.CreateWsIndexer(cfg) + wsClient, err := factory.CreateWsIndexer(cfg, clusterCfg) if err != nil { log.Error("cannot create ws indexer", "error", err) } @@ -87,16 +92,21 @@ func startIndexer(ctx *cli.Context) error { return nil } -func loadMainConfig(filepath string) (*config.Config, error) { - cfg := &config.Config{} - err := core.LoadTomlFile(cfg, filepath) - if err != nil { - return nil, err - } - return cfg, nil +func loadMainConfig(filepath string) (config.Config, error) { + cfg := config.Config{} + err := core.LoadTomlFile(&cfg, filepath) + + return cfg, err } -func initializeLogger(ctx *cli.Context, cfg *config.Config) (file.FileLoggingHandler, error) { +func loadClusterConfig(filepath string) (config.ClusterConfig, error) { + cfg := config.ClusterConfig{} + err := core.LoadTomlFile(&cfg, filepath) + + return cfg, err +} + +func initializeLogger(ctx *cli.Context, cfg config.Config) (file.FileLoggingHandler, error) { logLevelFlagValue := ctx.GlobalString(logLevel.Name) err := logger.SetLogLevel(logLevelFlagValue) if err != nil { @@ -131,5 +141,15 @@ func initializeLogger(ctx *cli.Context, cfg *config.Config) (file.FileLoggingHan return nil, err } + err = logger.RemoveLogObserver(os.Stdout) + if err != nil { + return nil, err + } + + err = logger.AddLogObserver(os.Stdout, &logger.PlainFormatter{}) + if err != nil { + return nil, err + } + return fileLogging, nil } diff --git a/config/config.go b/config/config.go index 5357093f..22866fd8 100644 --- a/config/config.go +++ b/config/config.go @@ -4,14 +4,7 @@ package config type Config struct { Config struct { EnabledIndices []string `toml:"enabled-indices"` - ElasticCluster struct { - UseKibana bool `toml:"use-kibana"` - URL string `toml:"url"` - UserName string `toml:"username"` - Password string `toml:"password"` - BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` - } `toml:"elastic-cluster"` - WebSocket struct { + WebSocket struct { ServerURL string `toml:"server-url"` DataMarshallerType string `toml:"data-marshaller-type"` } `toml:"web-socket"` @@ -40,3 +33,14 @@ type Config struct { } `toml:"logs"` } `toml:"config"` } + +// ClusterConfig will hold the config for the Elasticsearch cluster +type ClusterConfig struct { + ElasticCluster struct { + UseKibana bool `toml:"use-kibana"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` + BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + } `toml:"elastic-cluster"` +} diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 46c5046b..24a80c98 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -13,8 +13,13 @@ import ( var log = logger.GetOrCreate("factory") -func CreateWsIndexer(cfg *config.Config) (wsindexer.WSClient, error) { - dataIndexer, err := createDataIndexer(cfg) +const ( + indexerCacheSize = 1 +) + +// CreateWsIndexer will create a new instance of wsindexer.WSClient +func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.WSClient, error) { + dataIndexer, err := createDataIndexer(cfg, clusterCfg) if err != nil { return nil, err } @@ -25,11 +30,14 @@ func CreateWsIndexer(cfg *config.Config) (wsindexer.WSClient, error) { } indexer, err := wsindexer.NewIndexer(wsMarshaller, dataIndexer) + if err != nil { + return nil, err + } - return wsclient.NewWebSocketClient(cfg.Config.WebSocket.ServerURL, indexer.GetFunctionsMap()) + return wsclient.New(cfg.Config.WebSocket.ServerURL, indexer.GetFunctionsMap()) } -func createDataIndexer(cfg *config.Config) (wsindexer.DataIndexer, error) { +func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.DataIndexer, error) { marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) if err != nil { return nil, err @@ -47,15 +55,14 @@ func createDataIndexer(cfg *config.Config) (wsindexer.DataIndexer, error) { return nil, err } - return factory.NewIndexer(&factory.ArgsIndexerFactory{ - UseKibana: cfg.Config.ElasticCluster.UseKibana, - // Todo check if this is needed - IndexerCacheSize: 1, + return factory.NewIndexer(factory.ArgsIndexerFactory{ + UseKibana: clusterCfg.ElasticCluster.UseKibana, + IndexerCacheSize: indexerCacheSize, Denomination: cfg.Config.Economics.Denomination, - BulkRequestMaxSize: cfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, - Url: cfg.Config.ElasticCluster.URL, - UserName: cfg.Config.ElasticCluster.UserName, - Password: cfg.Config.ElasticCluster.Password, + BulkRequestMaxSize: clusterCfg.ElasticCluster.BulkRequestMaxSizeInBytes, + Url: clusterCfg.ElasticCluster.URL, + UserName: clusterCfg.ElasticCluster.UserName, + Password: clusterCfg.ElasticCluster.Password, EnabledIndexes: cfg.Config.EnabledIndices, Marshalizer: marshaller, Hasher: hasher, diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index bf63121e..69157ff2 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -68,7 +68,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids := []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} @@ -124,7 +124,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids = []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 45f75907..21ac01b6 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -79,7 +79,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262"} @@ -102,7 +102,7 @@ func TestIndexAccountsBalance(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -157,7 +157,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -189,7 +189,7 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262"} diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index c41d3894..24c2f77a 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -79,7 +79,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262-TOKEN-eeee-02"} diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index f9d52582..579ab5d4 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -54,7 +54,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -117,7 +117,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262-SEMI-abcd-02"} @@ -190,7 +190,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262-TTTT-abcd-02"} @@ -229,7 +229,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids = []string{"TTTT-abcd"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index 243ce5bd..caca7aec 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -112,7 +112,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index e2b24518..21cf894f 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -53,7 +53,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) // ################ CREATE SEMI FUNGIBLE TOKEN 1 ########################## @@ -129,7 +129,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids := []string{"61616161626262626363636363636363"} genericResponse := &GenericResponse{} @@ -159,7 +159,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 22 coreAlteredAccounts[addrForLogHex].Tokens[0].Nonce = 22 - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} @@ -208,7 +208,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} diff --git a/integrationtests/consts.go b/integrationtests/consts.go new file mode 100644 index 00000000..535f1ec1 --- /dev/null +++ b/integrationtests/consts.go @@ -0,0 +1,6 @@ +package integrationtests + +const ( + testsNumOfShards = 3 + esURL = "http://localhost:9200" +) diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index ffce5ce4..6bd64077 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -86,7 +86,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body := &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262-DESK-abcd-01"} @@ -133,7 +133,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 2 body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) genericResponse = &GenericResponse{} @@ -181,7 +181,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, 3) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) require.Nil(t, err) ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 382f6f7d..b84d412a 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -94,7 +94,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 1e255134..e2d5c9af 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -54,7 +54,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -83,7 +83,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -116,7 +116,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -145,7 +145,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -191,7 +191,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -219,7 +219,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index c6d0d862..199d4181 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -49,7 +49,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{"SSSS-abcd"} @@ -78,7 +78,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids = []string{"SSSS-abcd"} diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 9ea560ae..521fadf3 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -50,7 +50,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString([]byte("cross-log"))} @@ -91,7 +91,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) @@ -124,7 +124,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, 3) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 29b25971..66d5b8ad 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -53,7 +53,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{"NON-abcd"} @@ -97,7 +97,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids = []string{"NON-abcd-02"} @@ -132,7 +132,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids = []string{"NON-abcd-02"} diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index df2fd2e6..74215135 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -80,7 +80,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -120,7 +120,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -214,7 +214,7 @@ func TestNFTTransferCrossShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -252,7 +252,7 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -318,7 +318,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -378,7 +378,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index e06f3c2d..13667084 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -78,7 +78,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index bba77d98..19fdecf9 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -81,7 +81,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -123,7 +123,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, 3) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -212,7 +212,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 31067f65..f9364e10 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -72,7 +72,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -141,7 +141,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 0ba671d7..1adc82fe 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -54,7 +54,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { string(txHash): tx, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index 8eb3d65b..b1de1de9 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -58,7 +58,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids := []string{"NFT-abcd-0e"} @@ -85,7 +85,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) // Add URIS 2 --- results should be the same @@ -106,7 +106,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) // Update attributes 1 @@ -133,7 +133,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -161,7 +161,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, 3) + err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 0e3d54c6..a0bd27bf 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -15,8 +15,6 @@ import ( "github.com/elastic/go-elasticsearch/v7" ) -const esURL = "http://localhost:9200" - func setLogLevelDebug() { _ = logger.SetLogLevel("process:DEBUG") } diff --git a/process/dataindexer/dataDispatcher_test.go b/process/dataindexer/dataDispatcher_test.go index 69b69173..56b00ff5 100644 --- a/process/dataindexer/dataDispatcher_test.go +++ b/process/dataindexer/dataDispatcher_test.go @@ -11,7 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" - workItems2 "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" ) @@ -58,17 +58,17 @@ func TestDataDispatcher_StartIndexDataClose(t *testing.T) { return nil }, } - dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) wg.Wait() require.True(t, called) - dispatcher.Add(workItems2.NewItemAccounts(elasticProc, 0, nil, 0)) + dispatcher.Add(workItems.NewItemAccounts(elasticProc, 0, nil, 0)) wg.Add(1) - dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) - dispatcher.Add(workItems2.NewItemRating(elasticProc, "", nil)) + dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRating(elasticProc, "", nil)) wg.Add(1) - dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) err = dispatcher.Close() require.NoError(t, err) } @@ -97,7 +97,7 @@ func TestDataDispatcher_Add(t *testing.T) { } start := time.Now() - dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) wg.Wait() timePassed := time.Since(start) @@ -133,7 +133,7 @@ func TestDataDispatcher_AddWithErrorShouldRetryTheReprocessing(t *testing.T) { } start := time.Now() - dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) wg.Wait() timePassed := time.Since(start) @@ -173,7 +173,7 @@ func TestDataDispatcher_Close(t *testing.T) { if count == 105 { w.Done() } - dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) time.Sleep(50 * time.Millisecond) } } @@ -204,6 +204,6 @@ func TestDataDispatcher_RecoverPanic(t *testing.T) { }, } - dispatcher.Add(workItems2.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) dispatcher.doDataDispatch(context.Background()) } diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index 87ebdf48..0e2a33a7 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -23,9 +23,6 @@ var ErrNegativeCacheSize = errors.New("negative cache size") // ErrEmptyEnabledIndexes signals that an empty slice of enables indexes has been provided var ErrEmptyEnabledIndexes = errors.New("empty enabled indexes slice") -// ErrNilShardCoordinator signals that a nil shard coordinator was provided -var ErrNilShardCoordinator = errors.New("nil shard coordinator") - // ErrCouldNotCreatePolicy signals that the index policy hasn't been created var ErrCouldNotCreatePolicy = errors.New("could not create policy") diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 38da2ef0..9a2d4995 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -55,7 +55,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { bp, _ := block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) mp, _ := miniblocks.NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) vp, _ := validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32), 0) - args := &logsevents.ArgsLogsAndEventsProcessor{ + args := logsevents.ArgsLogsAndEventsProcessor{ PubKeyConverter: &mock.PubkeyConverterMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: balanceConverter, diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 675fc5c1..acb2c02b 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -86,7 +86,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E return nil, err } - argsLogsAndEventsProc := &logsevents.ArgsLogsAndEventsProcessor{ + argsLogsAndEventsProc := logsevents.ArgsLogsAndEventsProcessor{ PubKeyConverter: arguments.AddressPubkeyConverter, Marshalizer: arguments.Marshalizer, BalanceConverter: balanceConverter, diff --git a/process/elasticproc/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go index b127d2e5..3542c962 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor.go @@ -40,13 +40,13 @@ func newESDTIssueProcessor(pubkeyConverter core.PubkeyConverter) *esdtIssueProce } } -func (iep *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { +func (eip *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { if args.selfShardID != core.MetachainShardId { return argOutputProcessEvent{} } identifierStr := string(args.event.GetIdentifier()) - _, ok := iep.issueOperationsIdentifiers[identifierStr] + _, ok := eip.issueOperationsIdentifiers[identifierStr] if !ok { return argOutputProcessEvent{} } @@ -75,7 +75,7 @@ func (iep *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputPro numDecimals = big.NewInt(0).SetBytes(topics[4]).Uint64() } - encodedAddr := iep.pubkeyConverter.Encode(args.event.GetAddress()) + encodedAddr := eip.pubkeyConverter.Encode(args.event.GetAddress()) tokenInfo := &data.TokenInfo{ Token: string(topics[0]), @@ -96,7 +96,7 @@ func (iep *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputPro } if identifierStr == transferOwnershipFunc && len(topics) >= numIssueLogTopics+1 { - newOwner := iep.pubkeyConverter.Encode(topics[4]) + newOwner := eip.pubkeyConverter.Encode(topics[4]) tokenInfo.TransferOwnership = true tokenInfo.CurrentOwner = newOwner tokenInfo.OwnersHistory[0].Address = newOwner diff --git a/process/elasticproc/logsevents/esdtIssueProcessor_test.go b/process/elasticproc/logsevents/esdtIssueProcessor_test.go index 34da0685..f80b9157 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor_test.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor_test.go @@ -83,3 +83,23 @@ func TestIssueESDTProcessor_TransferOwnership(t *testing.T) { Properties: &data.TokenProperties{}, }, res.tokenInfo) } + +func TestIssueESDTProcessor_EventWithShardID0ShouldBeIgnored(t *testing.T) { + t.Parallel() + + esdtIssueProc := newESDTIssueProcessor(&mock.PubkeyConverterMock{}) + + event := &transaction.Event{ + Address: []byte("addr"), + Identifier: []byte(transferOwnershipFunc), + Topics: [][]byte{[]byte("MYTOKEN-abcd"), []byte("my-token"), []byte("MYTOKEN"), []byte(core.NonFungibleESDT), []byte("newOwner")}, + } + args := &argsProcessEvent{ + timestamp: 1234, + event: event, + selfShardID: 0, + } + + res := esdtIssueProc.processEvent(args) + require.False(t, res.processed) +} diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 2526f58f..e66a5b7e 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -30,7 +30,7 @@ type logsAndEventsProcessor struct { } // NewLogsAndEventsProcessor will create a new instance for the logsAndEventsProcessor -func NewLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) (*logsAndEventsProcessor, error) { +func NewLogsAndEventsProcessor(args ArgsLogsAndEventsProcessor) (*logsAndEventsProcessor, error) { err := checkArgsLogsAndEventsProcessor(args) if err != nil { return nil, err @@ -45,7 +45,7 @@ func NewLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) (*logsAndEvents }, nil } -func checkArgsLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) error { +func checkArgsLogsAndEventsProcessor(args ArgsLogsAndEventsProcessor) error { if check.IfNil(args.PubKeyConverter) { return dataindexer.ErrNilPubkeyConverter } @@ -62,7 +62,7 @@ func checkArgsLogsAndEventsProcessor(args *ArgsLogsAndEventsProcessor) error { return nil } -func createEventsProcessors(args *ArgsLogsAndEventsProcessor) []eventsProcessor { +func createEventsProcessors(args ArgsLogsAndEventsProcessor) []eventsProcessor { nftsProc := newNFTsProcessor(args.PubKeyConverter, args.Marshalizer) fungibleProc := newFungibleESDTProcessor(args.PubKeyConverter) scDeploysProc := newSCDeploysProcessor(args.PubKeyConverter) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 8c5963ec..ff4079cc 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -15,9 +15,9 @@ import ( "github.com/stretchr/testify/require" ) -func createMockArgs() *ArgsLogsAndEventsProcessor { +func createMockArgs() ArgsLogsAndEventsProcessor { balanceConverter, _ := converters.NewBalanceConverter(10) - return &ArgsLogsAndEventsProcessor{ + return ArgsLogsAndEventsProcessor{ PubKeyConverter: &mock.PubkeyConverterMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: balanceConverter, diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index 8b7a202f..7ce43743 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -5,7 +5,7 @@ import ( "time" "github.com/ElrondNetwork/elastic-indexer-go/data" - converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/sharding" coreData "github.com/ElrondNetwork/elrond-go-core/data" @@ -67,7 +67,7 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv } token := string(topics[0]) - identifier := converters2.ComputeTokenIdentifier(token, nonceBig.Uint64()) + identifier := converters.ComputeTokenIdentifier(token, nonceBig.Uint64()) valueBig := big.NewInt(0).SetBytes(topics[2]) if !np.shouldAddReceiverData(args) { @@ -135,7 +135,7 @@ func (np *nftsProcessor) processNFTEventOnSender( if eventIdentifier == core.BuiltInFunctionESDTNFTBurn || eventIdentifier == core.BuiltInFunctionESDTWipe { tokensSupply.Add(&data.TokenInfo{ Token: token, - Identifier: converters2.ComputeTokenIdentifier(token, nonceBig.Uint64()), + Identifier: converters.ComputeTokenIdentifier(token, nonceBig.Uint64()), Timestamp: time.Duration(timestamp), Nonce: nonceBig.Uint64(), }) @@ -163,10 +163,10 @@ func (np *nftsProcessor) processNFTEventOnSender( return } - tokenMetaData := converters2.PrepareTokenMetaData(np.pubKeyConverter, esdtToken) + tokenMetaData := converters.PrepareTokenMetaData(np.pubKeyConverter, esdtToken) tokensCreateInfo.Add(&data.TokenInfo{ Token: token, - Identifier: converters2.ComputeTokenIdentifier(token, nonceBig.Uint64()), + Identifier: converters.ComputeTokenIdentifier(token, nonceBig.Uint64()), Timestamp: time.Duration(timestamp), Data: tokenMetaData, Nonce: nonceBig.Uint64(), diff --git a/process/elasticproc/logsevents/serialize.go b/process/elasticproc/logsevents/serialize.go index 1c58db1f..58e0f111 100644 --- a/process/elasticproc/logsevents/serialize.go +++ b/process/elasticproc/logsevents/serialize.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/ElrondNetwork/elastic-indexer-go/data" - converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tokeninfo" "github.com/ElrondNetwork/elrond-go-core/core" ) @@ -14,7 +14,7 @@ import ( // SerializeLogs will serialize the provided logs in a way that Elasticsearch expects a bulk request func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data.BufferSlice, index string) error { for _, lg := range logs { - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(lg.ID), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(lg.ID), "\n")) serializedData, errMarshal := json.Marshal(lg) if errMarshal != nil { return errMarshal @@ -38,7 +38,7 @@ func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data. `"lang": "painless",`+ `"params": { "log": %s }},`+ `"upsert": {}}`, - converters2.FormatPainlessSource(codeToExecute), serializedData, + converters.FormatPainlessSource(codeToExecute), serializedData, ) err := buffSlice.PutData(meta, []byte(serializedDataStr)) @@ -53,7 +53,7 @@ func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data. // SerializeSCDeploys will serialize the provided smart contract deploys in a way that Elasticsearch expects a bulk request func (*logsAndEventsProcessor) SerializeSCDeploys(deploys map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice, index string) error { for scAddr, deployInfo := range deploys { - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(scAddr), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(scAddr), "\n")) serializedData, err := serializeDeploy(deployInfo) if err != nil { @@ -98,7 +98,7 @@ func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) { `"lang": "painless",`+ `"params": {"elem": %s}},`+ `"upsert": %s}`, - converters2.FormatPainlessSource(codeToExecute), string(upgradeSerialized), string(serializedData)) + converters.FormatPainlessSource(codeToExecute), string(upgradeSerialized), string(serializedData)) return []byte(serializedDataStr), nil } @@ -117,7 +117,7 @@ func (*logsAndEventsProcessor) SerializeTokens(tokens []*data.TokenInfo, updateN } } - return converters2.PrepareNFTUpdateData(buffSlice, updateNFTData, false, index) + return converters.PrepareNFTUpdateData(buffSlice, updateNFTData, false, index) } func serializeToken(tokenData *data.TokenInfo, index string) ([]byte, []byte, error) { @@ -125,7 +125,7 @@ func serializeToken(tokenData *data.TokenInfo, index string) ([]byte, []byte, er return serializeTokenTransferOwnership(tokenData, index) } - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tokenData.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tokenData.Token), "\n")) serializedTokenData, err := json.Marshal(tokenData) if err != nil { return nil, nil, err @@ -143,13 +143,13 @@ func serializeToken(tokenData *data.TokenInfo, index string) ([]byte, []byte, er `"lang": "painless",`+ `"params": {"token": %s}},`+ `"upsert": %s}`, - converters2.FormatPainlessSource(codeToExecute), string(serializedTokenData), string(serializedTokenData)) + converters.FormatPainlessSource(codeToExecute), string(serializedTokenData), string(serializedTokenData)) return meta, []byte(serializedDataStr), nil } func serializeTokenTransferOwnership(tokenData *data.TokenInfo, index string) ([]byte, []byte, error) { - meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tokenData.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tokenData.Token), "\n")) tokenDataSerialized, err := json.Marshal(tokenData) if err != nil { return nil, nil, err @@ -178,7 +178,7 @@ func serializeTokenTransferOwnership(tokenData *data.TokenInfo, index string) ([ `"lang": "painless",`+ `"params": {"elem": %s, "owner": "%s"}},`+ `"upsert": %s}`, - converters2.FormatPainlessSource(codeToExecute), string(ownerDataSerialized), converters2.JsonEscape(tokenData.CurrentOwner), string(tokenDataSerialized)) + converters.FormatPainlessSource(codeToExecute), string(ownerDataSerialized), converters.JsonEscape(tokenData.CurrentOwner), string(tokenDataSerialized)) return meta, []byte(serializedDataStr), nil } @@ -203,11 +203,11 @@ func (lep *logsAndEventsProcessor) SerializeDelegators(delegators map[string]*da func (lep *logsAndEventsProcessor) prepareSerializedDelegator(delegator *data.Delegator, index string) ([]byte, []byte, error) { id := lep.computeDelegatorID(delegator) if delegator.ShouldDelete { - meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(id), "\n")) + meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(id), "\n")) return meta, nil, nil } - meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(id), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(id), "\n")) serializedData, errMarshal := json.Marshal(delegator) if errMarshal != nil { return nil, nil, errMarshal @@ -231,7 +231,7 @@ func (lep *logsAndEventsProcessor) SerializeSupplyData(tokensSupply data.TokensH continue } - meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(supplyData.Identifier), "\n")) + meta := []byte(fmt.Sprintf(`{ "delete" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(supplyData.Identifier), "\n")) err := buffSlice.PutData(meta, nil) if err != nil { return err @@ -267,7 +267,7 @@ func (lep *logsAndEventsProcessor) SerializeRolesData( } func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role string, index string) error { - meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(rd.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(rd.Token), "\n")) var serializedDataStr string if rd.Set { codeToExecute := ` @@ -291,11 +291,11 @@ func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role `"lang": "painless",`+ `"params": { "role": "%s", "address": "%s"}},`+ `"upsert": { "roles": {"%s": ["%s"]}}}`, - converters2.FormatPainlessSource(codeToExecute), - converters2.JsonEscape(role), - converters2.JsonEscape(rd.Address), - converters2.JsonEscape(role), - converters2.JsonEscape(rd.Address), + converters.FormatPainlessSource(codeToExecute), + converters.JsonEscape(role), + converters.JsonEscape(rd.Address), + converters.JsonEscape(role), + converters.JsonEscape(rd.Address), ) } else { codeToExecute := ` @@ -310,9 +310,9 @@ func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role `"lang": "painless",`+ `"params": { "role": "%s", "address": "%s" }},`+ `"upsert": {} }`, - converters2.FormatPainlessSource(codeToExecute), - converters2.JsonEscape(role), - converters2.JsonEscape(rd.Address), + converters.FormatPainlessSource(codeToExecute), + converters.JsonEscape(role), + converters.JsonEscape(rd.Address), ) } @@ -320,7 +320,7 @@ func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role } func serializePropertiesData(buffSlice *data.BufferSlice, index string, tokenProp *tokeninfo.PropertiesData) error { - meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tokenProp.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : {"_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tokenProp.Token), "\n")) propertiesBytes, err := json.Marshal(tokenProp.Properties) if err != nil { @@ -340,7 +340,7 @@ func serializePropertiesData(buffSlice *data.BufferSlice, index string, tokenPro `"lang": "painless",`+ `"params": { "properties": %s}},`+ `"upsert": {}}}`, - converters2.FormatPainlessSource(codeToExecute), propertiesBytes) + converters.FormatPainlessSource(codeToExecute), propertiesBytes) return buffSlice.PutData(meta, []byte(serializedDataStr)) } diff --git a/process/elasticproc/tags/serialize.go b/process/elasticproc/tags/serialize.go index a5707ac1..4149eeba 100644 --- a/process/elasticproc/tags/serialize.go +++ b/process/elasticproc/tags/serialize.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/ElrondNetwork/elastic-indexer-go/data" - converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" ) // Serialize will serialize tagsCount in a way that Elasticsearch expects a bulk request @@ -16,14 +16,14 @@ func (tc *tagsCount) Serialize(buffSlice *data.BufferSlice, index string) error } base64Tag := base64.StdEncoding.EncodeToString([]byte(tag)) - meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(base64Tag), "\n")) + meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(base64Tag), "\n")) codeToExecute := ` ctx._source.count += params.count; ctx._source.tag = params.tag ` serializedDataStr := fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"count": %d, "tag": "%s"}},"upsert": {"count": %d, "tag":"%s"}}`, - converters2.FormatPainlessSource(codeToExecute), count, converters2.JsonEscape(tag), count, converters2.JsonEscape(tag), + converters.FormatPainlessSource(codeToExecute), count, converters.JsonEscape(tag), count, converters.JsonEscape(tag), ) err := buffSlice.PutData(meta, []byte(serializedDataStr)) diff --git a/process/elasticproc/transactions/serialize.go b/process/elasticproc/transactions/serialize.go index c5770327..ad9ece30 100644 --- a/process/elasticproc/transactions/serialize.go +++ b/process/elasticproc/transactions/serialize.go @@ -6,14 +6,14 @@ import ( "strings" "github.com/ElrondNetwork/elastic-indexer-go/data" - converters2 "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" + "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" ) // SerializeScResults will serialize the provided smart contract results in a way that ElasticSearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeScResults(scResults []*data.ScResult, buffSlice *data.BufferSlice, index string) error { for _, sc := range scResults { - meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(sc.Hash), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(sc.Hash), "\n")) serializedData, errPrepareSc := json.Marshal(sc) if errPrepareSc != nil { return errPrepareSc @@ -31,7 +31,7 @@ func (tdp *txsDatabaseProcessor) SerializeScResults(scResults []*data.ScResult, // SerializeReceipts will serialize the receipts in a way that ElasticSearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeReceipts(receipts []*data.Receipt, buffSlice *data.BufferSlice, index string) error { for _, rec := range receipts { - meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(rec.Hash), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index": "%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(rec.Hash), "\n")) serializedData, errPrepareReceipt := json.Marshal(rec) if errPrepareReceipt != nil { return errPrepareReceipt @@ -49,7 +49,7 @@ func (tdp *txsDatabaseProcessor) SerializeReceipts(receipts []*data.Receipt, buf // SerializeTransactionsFeeData will serialize transactions fee data func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[string]*data.FeeData, buffSlice *data.BufferSlice, index string) error { for txHash, feeData := range txHashRefund { - meta := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, converters2.JsonEscape(txHash), "\n")) + meta := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, converters.JsonEscape(txHash), "\n")) codeToExecute := ` if ('create' == ctx.op) { ctx.op = 'noop' @@ -64,7 +64,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s `"lang": "painless",`+ `"params": {"fee": "%s", "gasUsed": %d}},`+ `"upsert": {}}`, - converters2.FormatPainlessSource(codeToExecute), feeData.Fee, feeData.GasUsed, + converters.FormatPainlessSource(codeToExecute), feeData.Fee, feeData.GasUsed, ) err := buffSlice.PutData(meta, []byte(serializedDataStr)) @@ -119,7 +119,7 @@ func serializeTxHashStatus(buffSlice *data.BufferSlice, txHashStatus map[string] codeToExecute := ` ctx._source.status = params.status ` - serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"status": "%s"}},"upsert": %s }`, converters2.FormatPainlessSource(codeToExecute), converters2.JsonEscape(status), string(marshaledTx))) + serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"status": "%s"}},"upsert": %s }`, converters.FormatPainlessSource(codeToExecute), converters.JsonEscape(status), string(marshaledTx))) err = buffSlice.PutData(metaData, serializedData) if err != nil { return err @@ -134,7 +134,7 @@ func prepareSerializedDataForATransaction( selfShardID uint32, index string, ) ([]byte, []byte, error) { - metaData := []byte(fmt.Sprintf(`{"update":{ "_index":"%s", "_id":"%s"}}%s`, index, converters2.JsonEscape(tx.Hash), "\n")) + metaData := []byte(fmt.Sprintf(`{"update":{ "_index":"%s", "_id":"%s"}}%s`, index, converters.JsonEscape(tx.Hash), "\n")) marshaledTx, err := json.Marshal(tx) if err != nil { return nil, nil, err @@ -159,7 +159,7 @@ func prepareSerializedDataForATransaction( } // transaction is intra-shard, invalid or cross-shard destination me - meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters2.JsonEscape(tx.Hash), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(tx.Hash), "\n")) return meta, marshaledTx, nil } @@ -175,7 +175,7 @@ func prepareNFTESDTTransferOrMultiESDTTransfer(marshaledTx []byte) ([]byte, erro } ` serializedData := []byte(fmt.Sprintf(`{"scripted_upsert": true, "script":{"source":"%s","lang": "painless","params":{"tx": %s}},"upsert":{}}`, - converters2.FormatPainlessSource(codeToExecute), string(marshaledTx))) + converters.FormatPainlessSource(codeToExecute), string(marshaledTx))) return serializedData, nil } diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index cee9fb8f..d9c32cfe 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -23,9 +23,8 @@ var log = logger.GetOrCreate("indexer/factory") // ArgsIndexerFactory holds all dependencies required by the data indexer factory in order to create // new instances type ArgsIndexerFactory struct { - Enabled bool - UseKibana bool - // move this in ArgSaveBlock + Enabled bool + UseKibana bool IndexerCacheSize int Denomination int BulkRequestMaxSize int @@ -41,7 +40,7 @@ type ArgsIndexerFactory struct { } // NewIndexer will create a new instance of Indexer -func NewIndexer(args *ArgsIndexerFactory) (dataindexer.Indexer, error) { +func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { err := checkDataIndexerParams(args) if err != nil { return nil, err @@ -75,7 +74,7 @@ func retryBackOff(attempt int) time.Duration { return d } -func createElasticProcessor(args *ArgsIndexerFactory) (dataindexer.ElasticProcessor, error) { +func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcessor, error) { databaseClient, err := client.NewElasticClient(elasticsearch.Config{ Addresses: []string{args.Url}, Username: args.UserName, @@ -103,7 +102,7 @@ func createElasticProcessor(args *ArgsIndexerFactory) (dataindexer.ElasticProces return factory.CreateElasticProcessor(argsElasticProcFac) } -func checkDataIndexerParams(arguments *ArgsIndexerFactory) error { +func checkDataIndexerParams(arguments ArgsIndexerFactory) error { if arguments.IndexerCacheSize < 0 { return dataindexer.ErrNegativeCacheSize } diff --git a/process/factory/indexerFactory_test.go b/process/factory/indexerFactory_test.go index 4ac70d85..cce2b21b 100644 --- a/process/factory/indexerFactory_test.go +++ b/process/factory/indexerFactory_test.go @@ -11,10 +11,10 @@ import ( "github.com/stretchr/testify/require" ) -func createMockIndexerFactoryArgs() *ArgsIndexerFactory { +func createMockIndexerFactoryArgs() ArgsIndexerFactory { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - return &ArgsIndexerFactory{ + return ArgsIndexerFactory{ Enabled: true, IndexerCacheSize: 100, Url: ts.URL, @@ -32,12 +32,12 @@ func createMockIndexerFactoryArgs() *ArgsIndexerFactory { func TestNewIndexerFactory(t *testing.T) { tests := []struct { name string - argsFunc func() *ArgsIndexerFactory + argsFunc func() ArgsIndexerFactory exError error }{ { name: "InvalidCacheSize", - argsFunc: func() *ArgsIndexerFactory { + argsFunc: func() ArgsIndexerFactory { args := createMockIndexerFactoryArgs() args.IndexerCacheSize = -1 return args @@ -46,7 +46,7 @@ func TestNewIndexerFactory(t *testing.T) { }, { name: "NilAddressPubkeyConverter", - argsFunc: func() *ArgsIndexerFactory { + argsFunc: func() ArgsIndexerFactory { args := createMockIndexerFactoryArgs() args.AddressPubkeyConverter = nil return args @@ -55,7 +55,7 @@ func TestNewIndexerFactory(t *testing.T) { }, { name: "NilValidatorPubkeyConverter", - argsFunc: func() *ArgsIndexerFactory { + argsFunc: func() ArgsIndexerFactory { args := createMockIndexerFactoryArgs() args.ValidatorPubkeyConverter = nil return args @@ -64,7 +64,7 @@ func TestNewIndexerFactory(t *testing.T) { }, { name: "NilMarshalizer", - argsFunc: func() *ArgsIndexerFactory { + argsFunc: func() ArgsIndexerFactory { args := createMockIndexerFactoryArgs() args.Marshalizer = nil return args @@ -73,7 +73,7 @@ func TestNewIndexerFactory(t *testing.T) { }, { name: "NilHasher", - argsFunc: func() *ArgsIndexerFactory { + argsFunc: func() ArgsIndexerFactory { args := createMockIndexerFactoryArgs() args.Hasher = nil return args @@ -82,7 +82,7 @@ func TestNewIndexerFactory(t *testing.T) { }, { name: "EmptyUrl", - argsFunc: func() *ArgsIndexerFactory { + argsFunc: func() ArgsIndexerFactory { args := createMockIndexerFactoryArgs() args.Url = "" return args @@ -91,7 +91,7 @@ func TestNewIndexerFactory(t *testing.T) { }, { name: "All arguments ok", - argsFunc: func() *ArgsIndexerFactory { + argsFunc: func() ArgsIndexerFactory { return createMockIndexerFactoryArgs() }, exError: nil, diff --git a/process/wsclient/client.go b/process/wsclient/client.go index 35d6fd5c..cd594ed4 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -27,19 +27,26 @@ var ( ) type client struct { - urlReceive string - actions map[data.OperationType]func(marshalledData []byte) error + urlReceive string + actions map[data.OperationType]func(marshalledData []byte) error + uint64ByteSliceConverter websocketOutportDriver.Uint64ByteSliceConverter } -func NewWebSocketClient(urlReceive string, actions map[data.OperationType]func(marshalledData []byte) error) (*client, error) { +// New will create a new instance of web-sockets client +func New( + urlReceive string, + actions map[data.OperationType]func(marshalledData []byte, + ) error) (*client, error) { urlReceiveData := url.URL{Scheme: "ws", Host: fmt.Sprintf(urlReceive), Path: "/operations"} return &client{ - actions: actions, - urlReceive: urlReceiveData.String(), + actions: actions, + urlReceive: urlReceiveData.String(), + uint64ByteSliceConverter: uint64ByteSlice.NewBigEndianConverter(), }, nil } +// Start will initialize the connection to the server and start to listen for messages func (c *client) Start() { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, os.Interrupt) @@ -122,13 +129,12 @@ func (c *client) listeningOnWebSocket(wsConnection *websocket.Conn) { } func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsConn) { - uint64ByteSliceConverter := uint64ByteSlice.NewBigEndianConverter() if len(payload) == 0 { log.Error("empty payload") return } - payloadParser, _ := websocketOutportDriver.NewWebSocketPayloadParser(uint64ByteSliceConverter) + payloadParser, _ := websocketOutportDriver.NewWebSocketPayloadParser(c.uint64ByteSliceConverter) payloadData, err := payloadParser.ExtractPayloadData(payload) if err != nil { log.Error("error while extracting payload data: " + err.Error()) @@ -152,7 +158,7 @@ func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsCo } if payloadData.WithAcknowledge { - counterBytes := uint64ByteSliceConverter.ToByteSlice(payloadData.Counter) + counterBytes := c.uint64ByteSliceConverter.ToByteSlice(payloadData.Counter) err = ackHandler.WriteMessage(websocket.BinaryMessage, counterBytes) if err != nil { log.Error("write acknowledge message", "error", err.Error()) diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 04fdc3a1..ed3bcde5 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -1,19 +1,34 @@ package wsindexer import ( + "errors" + + "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" logger "github.com/ElrondNetwork/elrond-go-logger" ) -var log = logger.GetOrCreate("process/wsindexer") +var ( + log = logger.GetOrCreate("process/wsindexer") + errNilDataIndexer = errors.New("nil data indexer") +) type indexer struct { marshaller marshal.Marshalizer di DataIndexer } +// NewIndexer will create a new instance of *indexer func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*indexer, error) { + if check.IfNil(marshaller) { + return nil, dataindexer.ErrNilMarshalizer + } + if check.IfNil(dataIndexer) { + return nil, errNilDataIndexer + } + return &indexer{ marshaller: marshaller, di: dataIndexer, diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index bcca47cf..115ddac6 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -5,11 +5,13 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/outport" ) +// WSClient defines what a websockets client should do type WSClient interface { Start() Close() } +// DataIndexer dines what a data indexer should do type DataIndexer interface { SaveBlock(args *outport.ArgsSaveBlockData) error RevertIndexedBlock(header data.HeaderHandler, body data.BodyHandler) error diff --git a/process/wsindexer/unmarsall.go b/process/wsindexer/unmarshal.go similarity index 100% rename from process/wsindexer/unmarsall.go rename to process/wsindexer/unmarshal.go diff --git a/process/wsindexer/unmarshall_test.go b/process/wsindexer/unmarshal_test.go similarity index 100% rename from process/wsindexer/unmarshall_test.go rename to process/wsindexer/unmarshal_test.go From 9b52471df9466090237f3a6f78ed41c24c945643 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 30 Sep 2022 15:33:11 +0300 Subject: [PATCH 57/96] implement close --- cmd/elasticindexer/main.go | 21 ++- factory/wsIndexerFactory.go | 2 +- go.mod | 2 +- go.sum | 4 +- integrationtests/consts.go | 4 +- integrationtests/utils.go | 1 + .../transactions/transactionDBBuilder.go | 14 -- process/wsclient/client.go | 124 +++++++++--------- process/wsindexer/indexer.go | 2 +- 9 files changed, 87 insertions(+), 87 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index da598221..4719515d 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -3,6 +3,8 @@ package main import ( "fmt" "os" + "os/signal" + "syscall" "time" "github.com/ElrondNetwork/elastic-indexer-go/config" @@ -81,15 +83,22 @@ func startIndexer(ctx *cli.Context) error { log.Error("cannot create ws indexer", "error", err) } - wsClient.Start() + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGKILL) - log.Info("closing indexer") - if !check.IfNil(fileLogging) { - err = fileLogging.Close() - log.LogIfError(err) + go wsClient.Start() + + select { + case <-interrupt: + wsClient.Close() + if !check.IfNil(fileLogging) { + err = fileLogging.Close() + log.LogIfError(err) + } + + return nil } - return nil } func loadMainConfig(filepath string) (config.Config, error) { diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 24a80c98..c5b3de6a 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -34,7 +34,7 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return nil, err } - return wsclient.New(cfg.Config.WebSocket.ServerURL, indexer.GetFunctionsMap()) + return wsclient.New(cfg.Config.WebSocket.ServerURL, indexer) } func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.DataIndexer, error) { diff --git a/go.mod b/go.mod index fedb56cf..4b920be4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220927113554-34ccab6be8c3 + github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index af25bf3b..8747f778 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220927113554-34ccab6be8c3 h1:20nr2EMpS10pYr2Te/DXwIiAH8C86YvtTMB4kxSfw9g= -github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220927113554-34ccab6be8c3/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 h1:ht6Poy66/oPd0sQjo20WQppRYY1aDkxV5YW1lIslWOs= +github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 535f1ec1..8d3c5f82 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -1,6 +1,8 @@ package integrationtests const ( + //nolint testsNumOfShards = 3 - esURL = "http://localhost:9200" + //nolint + esURL = "http://localhost:9200" ) diff --git a/integrationtests/utils.go b/integrationtests/utils.go index a0bd27bf..b9e55676 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -19,6 +19,7 @@ func setLogLevelDebug() { _ = logger.SetLogLevel("process:DEBUG") } +//nolint func createESClient(url string) (elasticproc.DatabaseClientHandler, error) { return client.NewElasticClient(elasticsearch.Config{ Addresses: []string{url}, diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index e29f7231..884f27b7 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -8,7 +8,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elrond-go-core/core" - "github.com/ElrondNetwork/elrond-go-core/core/sharding" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/receipt" @@ -125,16 +124,3 @@ func (dtb *dbTransactionBuilder) prepareReceipt( Timestamp: time.Duration(header.GetTimeStamp()), } } - -func (dtb *dbTransactionBuilder) isInSameShard( - sender string, - selfShardID uint32, - numOfShard uint32, -) bool { - senderBytes, err := dtb.addressPubkeyConverter.Decode(sender) - if err != nil { - return false - } - - return sharding.ComputeShardID(senderBytes, numOfShard) == selfShardID -} diff --git a/process/wsclient/client.go b/process/wsclient/client.go index cd594ed4..d4526442 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -4,10 +4,10 @@ import ( "fmt" "io" "net/url" - "os" - "os/signal" + "strings" "time" + "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data/typeConverters/uint64ByteSlice" "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver" "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" @@ -15,6 +15,13 @@ import ( "github.com/gorilla/websocket" ) +const closedConnection = "use of closed network connection" + +type operationsHandler interface { + GetOperationsMap() map[data.OperationType]func(marshalledData []byte) error + Close() error +} + type wsConn interface { io.Closer ReadMessage() (messageType int, p []byte, err error) @@ -28,19 +35,22 @@ var ( type client struct { urlReceive string + closeActions func() error actions map[data.OperationType]func(marshalledData []byte) error uint64ByteSliceConverter websocketOutportDriver.Uint64ByteSliceConverter + wsConnection wsConn } // New will create a new instance of web-sockets client func New( urlReceive string, - actions map[data.OperationType]func(marshalledData []byte, - ) error) (*client, error) { - urlReceiveData := url.URL{Scheme: "ws", Host: fmt.Sprintf(urlReceive), Path: "/operations"} + operationsHandler operationsHandler, +) (*client, error) { + urlReceiveData := url.URL{Scheme: "ws", Host: fmt.Sprintf(urlReceive), Path: data.WSRoute} return &client{ - actions: actions, + actions: operationsHandler.GetOperationsMap(), + closeActions: operationsHandler.Close, urlReceive: urlReceiveData.String(), uint64ByteSliceConverter: uint64ByteSlice.NewBigEndianConverter(), }, nil @@ -48,87 +58,56 @@ func New( // Start will initialize the connection to the server and start to listen for messages func (c *client) Start() { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - log.Info("connecting to", "url", c.urlReceive) - var wsConnection *websocket.Conn - done := make(chan struct{}) - go func() { - defer close(done) - for { - var err error - wsConnection, err = c.openConnection() - if err != nil { - log.Warn(fmt.Sprintf("websocket error, retrying in %v...", retryDuration), "error", err.Error()) - time.Sleep(retryDuration) - continue - } - - c.listeningOnWebSocket(wsConnection) + for { + err := c.openConnection() + if err != nil { + log.Warn(fmt.Sprintf("c.openConnection(), retrying in %v...", retryDuration), "error", err.Error()) time.Sleep(retryDuration) + continue } - }() - - timer := time.NewTimer(time.Second) - defer timer.Stop() - for { - select { - case <-done: - return - case <-timer.C: - case <-interrupt: - log.Info("interrupt") - if wsConnection == nil { - return - } - - // Cleanly close the connection by sending a close message and then - // waiting (with timeout) for the server to close the connection. - err := wsConnection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) - if err != nil { - log.Error("write close", "error", err) - return - } - select { - case <-done: - case <-time.After(time.Second): - } + closed := c.listeningOnWebSocket() + if closed { return } } } -func (c *client) openConnection() (*websocket.Conn, error) { - conn, _, err := websocket.DefaultDialer.Dial(c.urlReceive, nil) +func (c *client) openConnection() error { + var err error + c.wsConnection, _, err = websocket.DefaultDialer.Dial(c.urlReceive, nil) if err != nil { - return nil, err + return err } - return conn, nil + return nil } -func (c *client) listeningOnWebSocket(wsConnection *websocket.Conn) { +func (c *client) listeningOnWebSocket() (closed bool) { for { - _, message, err := wsConnection.ReadMessage() + _, message, err := c.wsConnection.ReadMessage() if err == nil { - c.verifyPayloadAndSendAckIfNeeded(message, wsConnection) + c.verifyPayloadAndSendAckIfNeeded(message) continue } _, isConnectionClosed := err.(*websocket.CloseError) if !isConnectionClosed { - log.Warn("websocket error, retrying in %v...", "error", err.Error()) + if strings.Contains(err.Error(), closedConnection) { + return true + } + log.Warn("websocket error, retrying in ...", "error", err.Error()) } else { log.Warn(fmt.Sprintf("websocket terminated by the server side, retrying in %v...", retryDuration), "error", err.Error()) } return } + } -func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsConn) { +func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte) { if len(payload) == 0 { log.Error("empty payload") return @@ -159,14 +138,37 @@ func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte, ackHandler wsCo if payloadData.WithAcknowledge { counterBytes := c.uint64ByteSliceConverter.ToByteSlice(payloadData.Counter) - err = ackHandler.WriteMessage(websocket.BinaryMessage, counterBytes) + err = c.wsConnection.WriteMessage(websocket.BinaryMessage, counterBytes) if err != nil { log.Error("write acknowledge message", "error", err.Error()) } } } +func (c *client) closeWsConnection() { + log.Debug("closing ws connection...") + if check.IfNilReflect(c.wsConnection) { + return + } + + //Cleanly close the connection by sending a close message and then + //waiting (with timeout) for the server to close the connection. + err := c.wsConnection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + if err != nil { + log.Error("cannot send close message", "error", err) + } + err = c.wsConnection.Close() + if err != nil { + log.Error("cannot close ws connection", "error", err) + } +} + func (c *client) Close() { - //TODO implement me - panic("implement me") + log.Info("closing all components...") + c.closeWsConnection() + + err := c.closeActions() + if err != nil { + log.Error("cannot close the operations handler", "error", err) + } } diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index ed3bcde5..ca68692c 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -35,7 +35,7 @@ func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*index }, nil } -func (i *indexer) GetFunctionsMap() map[data.OperationType]func(d []byte) error { +func (i *indexer) GetOperationsMap() map[data.OperationType]func(d []byte) error { return map[data.OperationType]func(d []byte) error{ data.OperationSaveBlock: i.saveBlock, data.OperationRevertIndexedBlock: i.revertIndexedBlock, From f0b4137aafc76600792f054c36f70bbab10a95bd Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 30 Sep 2022 16:06:39 +0300 Subject: [PATCH 58/96] fix linter issues --- cmd/elasticindexer/main.go | 18 ++++++-------- process/wsclient/client.go | 2 +- process/wsindexer/types.go | 43 ++++++++++++++++++++++++++++++++++ process/wsindexer/unmarshal.go | 38 +----------------------------- 4 files changed, 52 insertions(+), 49 deletions(-) create mode 100644 process/wsindexer/types.go diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 4719515d..00d1639b 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -84,21 +84,17 @@ func startIndexer(ctx *cli.Context) error { } interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, syscall.SIGINT, syscall.SIGKILL) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) go wsClient.Start() - select { - case <-interrupt: - wsClient.Close() - if !check.IfNil(fileLogging) { - err = fileLogging.Close() - log.LogIfError(err) - } - - return nil + <-interrupt + wsClient.Close() + if !check.IfNil(fileLogging) { + err = fileLogging.Close() + log.LogIfError(err) } - + return nil } func loadMainConfig(filepath string) (config.Config, error) { diff --git a/process/wsclient/client.go b/process/wsclient/client.go index d4526442..5f7d1524 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -46,7 +46,7 @@ func New( urlReceive string, operationsHandler operationsHandler, ) (*client, error) { - urlReceiveData := url.URL{Scheme: "ws", Host: fmt.Sprintf(urlReceive), Path: data.WSRoute} + urlReceiveData := url.URL{Scheme: "ws", Host: urlReceive, Path: data.WSRoute} return &client{ actions: operationsHandler.GetOperationsMap(), diff --git a/process/wsindexer/types.go b/process/wsindexer/types.go new file mode 100644 index 00000000..f72f4f61 --- /dev/null +++ b/process/wsindexer/types.go @@ -0,0 +1,43 @@ +package wsindexer + +import ( + "github.com/ElrondNetwork/elrond-go-core/data/outport" + "github.com/ElrondNetwork/elrond-go-core/data/receipt" + "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" + "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" + "github.com/ElrondNetwork/elrond-go-core/data/transaction" +) + +type normalTxWrapped struct { + TransactionHandler *transaction.Transaction + outport.FeeInfo +} +type rewardsTxsWrapped struct { + TransactionHandler *rewardTx.RewardTx + outport.FeeInfo +} +type scrWrapped struct { + TransactionHandler *smartContractResult.SmartContractResult + outport.FeeInfo +} +type receiptWrapped struct { + TransactionHandler *receipt.Receipt + outport.FeeInfo +} +type logWrapped struct { + TxHash string + LogHandler *transaction.Log +} + +type poolStruct struct { + Txs map[string]*normalTxWrapped + Invalid map[string]*normalTxWrapped + Scrs map[string]*scrWrapped + Rewards map[string]*rewardsTxsWrapped + Receipts map[string]*receiptWrapped + Logs []*logWrapped +} + +type argsSaveBlock struct { + TransactionsPool *poolStruct +} diff --git a/process/wsindexer/unmarshal.go b/process/wsindexer/unmarshal.go index 3a420058..81a09eab 100644 --- a/process/wsindexer/unmarshal.go +++ b/process/wsindexer/unmarshal.go @@ -8,10 +8,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" - "github.com/ElrondNetwork/elrond-go-core/data/receipt" - "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" - "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" - "github.com/ElrondNetwork/elrond-go-core/data/transaction" outportData "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" ) @@ -64,39 +60,7 @@ func (i *indexer) getArgsSaveBlock(marshaledData []byte) (*outport.ArgsSaveBlock } func (i *indexer) getTxsPool(marshaledData []byte) (*outport.Pool, error) { - type normalTxWrapped struct { - TransactionHandler *transaction.Transaction - outport.FeeInfo - } - type rewardsTxsWrapped struct { - TransactionHandler *rewardTx.RewardTx - outport.FeeInfo - } - type scrWrapped struct { - TransactionHandler *smartContractResult.SmartContractResult - outport.FeeInfo - } - type receiptWrapped struct { - TransactionHandler *receipt.Receipt - outport.FeeInfo - } - type logWrapped struct { - TxHash string - LogHandler *transaction.Log - } - - type poolStruct struct { - Txs map[string]*normalTxWrapped - Invalid map[string]*normalTxWrapped - Scrs map[string]*scrWrapped - Rewards map[string]*rewardsTxsWrapped - Receipts map[string]*receiptWrapped - Logs []*logWrapped - } - - argSaveBlock := struct { - TransactionsPool *poolStruct - }{} + argSaveBlock := argsSaveBlock{} err := i.marshaller.Unmarshal(&argSaveBlock, marshaledData) if err != nil { From d8a3f18cacf19d877dce3076903894c97b6f766c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Oct 2022 12:51:09 +0300 Subject: [PATCH 59/96] latest elrond go logger --- cmd/elasticindexer/main.go | 5 +++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 00d1639b..a4f25b3e 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -11,6 +11,7 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/factory" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" + "github.com/ElrondNetwork/elrond-go-core/core/closing" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/ElrondNetwork/elrond-go-logger/file" "github.com/urfave/cli" @@ -90,7 +91,7 @@ func startIndexer(ctx *cli.Context) error { <-interrupt wsClient.Close() - if !check.IfNil(fileLogging) { + if !check.IfNilReflect(fileLogging) { err = fileLogging.Close() log.LogIfError(err) } @@ -111,7 +112,7 @@ func loadClusterConfig(filepath string) (config.ClusterConfig, error) { return cfg, err } -func initializeLogger(ctx *cli.Context, cfg config.Config) (file.FileLoggingHandler, error) { +func initializeLogger(ctx *cli.Context, cfg config.Config) (closing.Closer, error) { logLevelFlagValue := ctx.GlobalString(logLevel.Name) err := logger.SetLogLevel(logLevelFlagValue) if err != nil { diff --git a/go.mod b/go.mod index 4b920be4..6a391a92 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 - github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 + github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20221003085013-fb326ba1620f github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 diff --git a/go.sum b/go.sum index 8747f778..1d7a3e7f 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 h1 github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= -github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23 h1:Mgw3Yfu0kll1i/tMsgwkxnCiPEueatb3j2giteOYMAA= -github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20220901113442-d5ad00505a23/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= +github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20221003085013-fb326ba1620f h1:JnlooT94jWuWQKkzdme3XHjViHvj9Ojhs3EMWC8m97g= +github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20221003085013-fb326ba1620f/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 h1:LM7cObmyt3ZyGXGxE8y5K/ME4SDO/ZuSaixQpsjWEYM= github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6/go.mod h1:7qdwuZaUr5JFiOABCVgAgiuMHEJ2B8Lv5cTTAFedZNA= From 14ddedc0ae3b82bc60e21b315c68d3a3d9095da2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Oct 2022 16:31:12 +0300 Subject: [PATCH 60/96] fixes after re review --- cmd/elasticindexer/config/config.toml | 8 ------ cmd/elasticindexer/config/prefs.toml | 22 +++++++++++----- cmd/elasticindexer/main.go | 13 +++++++--- config/config.go | 26 ++++++++++--------- factory/wsIndexerFactory.go | 16 ++++++------ go.mod | 2 +- go.sum | 4 +-- .../accountsBalanceNftTransfer_test.go | 4 +-- .../accountsBalanceWithLowerTimestamp_test.go | 8 +++--- integrationtests/accountsESDTRollback_test.go | 2 +- .../accountsESDTWithTokenType_test.go | 8 +++--- integrationtests/claimRewards_test.go | 2 +- integrationtests/collectionsIndex_test.go | 8 +++--- integrationtests/consts.go | 2 +- integrationtests/createNFTWithTags_test.go | 6 ++--- integrationtests/esdtTransfer_test.go | 2 +- .../issueTokenAndSetRoles_test.go | 12 ++++----- integrationtests/issueToken_test.go | 4 +-- integrationtests/logsCrossShard_test.go | 6 ++--- integrationtests/nftIssueCreateBurn_test.go | 6 ++--- .../nftTransferCrossShard_test.go | 12 ++++----- .../nftTransferCrossWithScCall_test.go | 2 +- integrationtests/relayedTx_test.go | 6 ++--- integrationtests/scCallIntraShard_test.go | 4 +-- integrationtests/transactions_test.go | 2 +- integrationtests/updateNFT_test.go | 10 +++---- process/wsclient/client.go | 4 +-- process/wsindexer/indexer.go | 5 ++-- process/wsindexer/interface.go | 2 +- 29 files changed, 109 insertions(+), 99 deletions(-) diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index 3a58f705..2423ab69 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -1,12 +1,4 @@ [config] - enabled-indices = [ - "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", - "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", - "logs", "delegators", "operations", "collections" - ] - [config.web-socket] - server-url = "localhost:22111" - data-marshaller-type = "json" [config.address-converter] length = 32 type = "bech32" diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index e134fd36..793fd8cb 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -1,6 +1,16 @@ -[elastic-cluster] - use-kibana = false - url = "http://localhost:9200" - username = "" - password = "" - bulk-request-max-size-in-bytes = 4194304 # 4MB +[config] + enabled-indices = [ + "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", + "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", + "logs", "delegators", "operations", "collections" + ] + [config.web-socket] + server-url = "localhost:22111" + data-marshaller-type = "json" + + [config.elastic-cluster] + use-kibana = false + url = "http://localhost:9200" + username = "" + password = "" + bulk-request-max-size-in-bytes = 4194304 # 4MB diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index a4f25b3e..7143c564 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -90,6 +90,7 @@ func startIndexer(ctx *cli.Context) error { go wsClient.Start() <-interrupt + log.Info("closing app at user's signal") wsClient.Close() if !check.IfNilReflect(fileLogging) { err = fileLogging.Close() @@ -147,15 +148,19 @@ func initializeLogger(ctx *cli.Context, cfg config.Config) (closing.Closer, erro return nil, err } - err = logger.RemoveLogObserver(os.Stdout) + err = removeANSIColorsForLogger() if err != nil { return nil, err } - err = logger.AddLogObserver(os.Stdout, &logger.PlainFormatter{}) + return fileLogging, nil +} + +func removeANSIColorsForLogger() error { + err := logger.RemoveLogObserver(os.Stdout) if err != nil { - return nil, err + return err } - return fileLogging, nil + return logger.AddLogObserver(os.Stdout, &logger.PlainFormatter{}) } diff --git a/config/config.go b/config/config.go index 22866fd8..02c589fc 100644 --- a/config/config.go +++ b/config/config.go @@ -3,11 +3,6 @@ package config // Config will hold the whole config file's data type Config struct { Config struct { - EnabledIndices []string `toml:"enabled-indices"` - WebSocket struct { - ServerURL string `toml:"server-url"` - DataMarshallerType string `toml:"data-marshaller-type"` - } `toml:"web-socket"` AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` @@ -36,11 +31,18 @@ type Config struct { // ClusterConfig will hold the config for the Elasticsearch cluster type ClusterConfig struct { - ElasticCluster struct { - UseKibana bool `toml:"use-kibana"` - URL string `toml:"url"` - UserName string `toml:"username"` - Password string `toml:"password"` - BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` - } `toml:"elastic-cluster"` + Config struct { + EnabledIndices []string `toml:"enabled-indices"` + WebSocket struct { + ServerURL string `toml:"server-url"` + DataMarshallerType string `toml:"data-marshaller-type"` + } `toml:"web-socket"` + ElasticCluster struct { + UseKibana bool `toml:"use-kibana"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` + BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + } `toml:"elastic-cluster"` + } `toml:"config"` } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index c5b3de6a..e0e2691c 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -24,7 +24,7 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return nil, err } - wsMarshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.WebSocket.DataMarshallerType) + wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) if err != nil { return nil, err } @@ -34,7 +34,7 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return nil, err } - return wsclient.New(cfg.Config.WebSocket.ServerURL, indexer) + return wsclient.New(clusterCfg.Config.WebSocket.ServerURL, indexer) } func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.DataIndexer, error) { @@ -56,14 +56,14 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin } return factory.NewIndexer(factory.ArgsIndexerFactory{ - UseKibana: clusterCfg.ElasticCluster.UseKibana, + UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, IndexerCacheSize: indexerCacheSize, Denomination: cfg.Config.Economics.Denomination, - BulkRequestMaxSize: clusterCfg.ElasticCluster.BulkRequestMaxSizeInBytes, - Url: clusterCfg.ElasticCluster.URL, - UserName: clusterCfg.ElasticCluster.UserName, - Password: clusterCfg.ElasticCluster.Password, - EnabledIndexes: cfg.Config.EnabledIndices, + BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, + Url: clusterCfg.Config.ElasticCluster.URL, + UserName: clusterCfg.Config.ElasticCluster.UserName, + Password: clusterCfg.Config.ElasticCluster.Password, + EnabledIndexes: clusterCfg.Config.EnabledIndices, Marshalizer: marshaller, Hasher: hasher, AddressPubkeyConverter: addressPubkeyConverter, diff --git a/go.mod b/go.mod index 6a391a92..93bb86dd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 - github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20221003085013-fb326ba1620f + github.com/ElrondNetwork/elrond-go-logger v1.0.8 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 diff --git a/go.sum b/go.sum index 1d7a3e7f..467e258d 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 h1 github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= -github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20221003085013-fb326ba1620f h1:JnlooT94jWuWQKkzdme3XHjViHvj9Ojhs3EMWC8m97g= -github.com/ElrondNetwork/elrond-go-logger v1.0.8-0.20221003085013-fb326ba1620f/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= +github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= +github.com/ElrondNetwork/elrond-go-logger v1.0.8/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 h1:LM7cObmyt3ZyGXGxE8y5K/ME4SDO/ZuSaixQpsjWEYM= github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6/go.mod h1:7qdwuZaUr5JFiOABCVgAgiuMHEJ2B8Lv5cTTAFedZNA= diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 69157ff2..41d010ca 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -68,7 +68,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids := []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} @@ -124,7 +124,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids = []string{"746573742d616464726573732d62616c616e63652d31-NFT-abcdef-718863"} diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 21ac01b6..66a4ad4f 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -79,7 +79,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262"} @@ -102,7 +102,7 @@ func TestIndexAccountsBalance(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -157,7 +157,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262"} @@ -189,7 +189,7 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262"} diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 24c2f77a..76e98e5c 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -79,7 +79,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262-TOKEN-eeee-02"} diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 579ab5d4..3eb28b83 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -54,7 +54,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -117,7 +117,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids = []string{"6161616162626262-SEMI-abcd-02"} @@ -190,7 +190,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262-TTTT-abcd-02"} @@ -229,7 +229,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids = []string{"TTTT-abcd"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index caca7aec..e09cd92d 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -112,7 +112,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index 21cf894f..e0297eae 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -53,7 +53,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) // ################ CREATE SEMI FUNGIBLE TOKEN 1 ########################## @@ -129,7 +129,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids := []string{"61616161626262626363636363636363"} genericResponse := &GenericResponse{} @@ -159,7 +159,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 22 coreAlteredAccounts[addrForLogHex].Tokens[0].Nonce = 22 - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} @@ -208,7 +208,7 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids = []string{"61616161626262626363636363636363"} genericResponse = &GenericResponse{} diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 8d3c5f82..6995ff9d 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -2,7 +2,7 @@ package integrationtests const ( //nolint - testsNumOfShards = 3 + testNumOfShards = 3 //nolint esURL = "http://localhost:9200" ) diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 6bd64077..cdb2329c 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -86,7 +86,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body := &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids := []string{"6161616162626262-DESK-abcd-01"} @@ -133,7 +133,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[addrHex].Tokens[0].Nonce = 2 body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) genericResponse = &GenericResponse{} @@ -181,7 +181,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) require.Nil(t, err) ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index b84d412a..797d6aa6 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -94,7 +94,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index e2d5c9af..d6f1522a 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -54,7 +54,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -83,7 +83,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -116,7 +116,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -145,7 +145,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -191,7 +191,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -219,7 +219,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 199d4181..15d7a536 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -49,7 +49,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{"SSSS-abcd"} @@ -78,7 +78,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids = []string{"SSSS-abcd"} diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 521fadf3..2fea3ab8 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -50,7 +50,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString([]byte("cross-log"))} @@ -91,7 +91,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) @@ -124,7 +124,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 66d5b8ad..a83b3d5e 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -53,7 +53,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{"NON-abcd"} @@ -97,7 +97,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids = []string{"NON-abcd-02"} @@ -132,7 +132,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids = []string{"NON-abcd-02"} diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 74215135..c0816643 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -80,7 +80,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -120,7 +120,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -214,7 +214,7 @@ func TestNFTTransferCrossShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -252,7 +252,7 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -318,7 +318,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -378,7 +378,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 13667084..0b8cb35a 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -78,7 +78,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 19fdecf9..669fc1c7 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -81,7 +81,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -123,7 +123,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -212,7 +212,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index f9364e10..000f007c 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -72,7 +72,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -141,7 +141,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { }, 0, big.NewInt(0)), }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 1adc82fe..18033bbb 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -54,7 +54,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { string(txHash): tx, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index b1de1de9..feb5fc19 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -58,7 +58,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids := []string{"NFT-abcd-0e"} @@ -85,7 +85,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) // Add URIS 2 --- results should be the same @@ -106,7 +106,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) // Update attributes 1 @@ -133,7 +133,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -161,7 +161,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testsNumOfShards) + err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} diff --git a/process/wsclient/client.go b/process/wsclient/client.go index 5f7d1524..dfc98724 100644 --- a/process/wsclient/client.go +++ b/process/wsclient/client.go @@ -41,7 +41,7 @@ type client struct { wsConnection wsConn } -// New will create a new instance of web-sockets client +// New will create a new instance of websocket client func New( urlReceive string, operationsHandler operationsHandler, @@ -98,7 +98,7 @@ func (c *client) listeningOnWebSocket() (closed bool) { if strings.Contains(err.Error(), closedConnection) { return true } - log.Warn("websocket error, retrying in ...", "error", err.Error()) + log.Warn("c.listeningOnWebSocket()-> connection problem, retrying", "error", err.Error()) } else { log.Warn(fmt.Sprintf("websocket terminated by the server side, retrying in %v...", retryDuration), "error", err.Error()) } diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index ca68692c..9d80791c 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -35,6 +35,7 @@ func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*index }, nil } +// GetOperationsMap returns the map with all the operations that will index data func (i *indexer) GetOperationsMap() map[data.OperationType]func(d []byte) error { return map[data.OperationType]func(d []byte) error{ data.OperationSaveBlock: i.saveBlock, @@ -48,12 +49,12 @@ func (i *indexer) GetOperationsMap() map[data.OperationType]func(d []byte) error } func (i *indexer) saveBlock(marshalledData []byte) error { - argsSaveBlock, err := i.getArgsSaveBlock(marshalledData) + argsSaveBlockS, err := i.getArgsSaveBlock(marshalledData) if err != nil { return err } - return i.di.SaveBlock(argsSaveBlock) + return i.di.SaveBlock(argsSaveBlockS) } func (i *indexer) revertIndexedBlock(marshalledData []byte) error { diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index 115ddac6..d20a70ba 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -5,7 +5,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/outport" ) -// WSClient defines what a websockets client should do +// WSClient defines what a websocket client should do type WSClient interface { Start() Close() From cd568fa0300fece9cdc8e28432c6bafcfddc2370 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 11:11:50 +0300 Subject: [PATCH 61/96] fixes --- .github/workflows/golangci-lint.yml | 3 +-- client/elasticClient.go | 2 +- cmd/elasticindexer/config/config.toml | 5 +++++ cmd/elasticindexer/config/prefs.toml | 5 ----- cmd/elasticindexer/flags.go | 5 +++++ cmd/elasticindexer/main.go | 10 ++++++++-- config/config.go | 4 ++-- factory/wsIndexerFactory.go | 2 +- 8 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index c63a71a9..989a0015 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -1,8 +1,7 @@ name: golangci-lint on: push: - branches: - - master + branches: [ master, development, feat/*, rc/* ] pull_request: branches: [ master, development, feat/*, rc/* ] diff --git a/client/elasticClient.go b/client/elasticClient.go index f1e18dbc..3b6fc950 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -310,7 +310,7 @@ func (ec *elasticClient) createPolicy(policyName string, policy *bytes.Buffer) e // CreateIndexTemplate creates an elasticsearch index template func (ec *elasticClient) createIndexTemplate(templateName string, template io.Reader) error { - res, err := ec.client.Indices.PutTemplate(templateName, template) + res, err := ec.client.Indices.PutTemplate(templateName, template, ec.client.Indices.PutTemplate.WithContext(context.Background())) if err != nil { return err } diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index 2423ab69..3272a0bc 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -1,4 +1,9 @@ [config] + enabled-indices = [ + "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", + "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", + "logs", "delegators", "operations", "collections" + ] [config.address-converter] length = 32 type = "bech32" diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 793fd8cb..8dac4249 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -1,9 +1,4 @@ [config] - enabled-indices = [ - "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", - "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", - "logs", "delegators", "operations", "collections" - ] [config.web-socket] server-url = "localhost:22111" data-marshaller-type = "json" diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index 489d6550..ef9cd750 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -30,4 +30,9 @@ var ( Name: "log-save", Usage: "Boolean option for enabling log saving. If set, it will automatically save all the logs into a file.", } + // disableAnsiColor defines if the logger subsystem should prevent displaying ANSI colors + disableAnsiColor = cli.BoolFlag{ + Name: "disable-ansi-color", + Usage: "Boolean option for disabling ANSI colors in the logging system.", + } ) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 7143c564..7c215577 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -46,6 +46,7 @@ func main() { configurationPreferencesFile, logLevel, logSaveFile, + disableAnsiColor, } app.Authors = []cli.Author{ { @@ -148,7 +149,8 @@ func initializeLogger(ctx *cli.Context, cfg config.Config) (closing.Closer, erro return nil, err } - err = removeANSIColorsForLogger() + disableAnsi := ctx.GlobalBool(disableAnsiColor.Name) + err = removeANSIColorsForLoggerIfNeeded(disableAnsi) if err != nil { return nil, err } @@ -156,7 +158,11 @@ func initializeLogger(ctx *cli.Context, cfg config.Config) (closing.Closer, erro return fileLogging, nil } -func removeANSIColorsForLogger() error { +func removeANSIColorsForLoggerIfNeeded(disableAnsi bool) error { + if !disableAnsi { + return nil + } + err := logger.RemoveLogObserver(os.Stdout) if err != nil { return err diff --git a/config/config.go b/config/config.go index 02c589fc..ae81d541 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config // Config will hold the whole config file's data type Config struct { Config struct { + EnabledIndices []string `toml:"enabled-indices"` AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` @@ -32,8 +33,7 @@ type Config struct { // ClusterConfig will hold the config for the Elasticsearch cluster type ClusterConfig struct { Config struct { - EnabledIndices []string `toml:"enabled-indices"` - WebSocket struct { + WebSocket struct { ServerURL string `toml:"server-url"` DataMarshallerType string `toml:"data-marshaller-type"` } `toml:"web-socket"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index e0e2691c..f9c4198e 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -63,7 +63,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin Url: clusterCfg.Config.ElasticCluster.URL, UserName: clusterCfg.Config.ElasticCluster.UserName, Password: clusterCfg.Config.ElasticCluster.Password, - EnabledIndexes: clusterCfg.Config.EnabledIndices, + EnabledIndexes: cfg.Config.EnabledIndices, Marshalizer: marshaller, Hasher: hasher, AddressPubkeyConverter: addressPubkeyConverter, From d9de7ca40acf910da9572b349c864e953a5c5697 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 11:55:59 +0300 Subject: [PATCH 62/96] available indices and disabled --- cmd/elasticindexer/config/config.toml | 2 +- cmd/elasticindexer/config/prefs.toml | 1 + config/config.go | 5 +++-- factory/wsIndexerFactory.go | 20 +++++++++++++++++++- factory/wsindexerFactory_test.go | 23 +++++++++++++++++++++++ 5 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 factory/wsindexerFactory_test.go diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index 3272a0bc..d1a9f2c7 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -1,5 +1,5 @@ [config] - enabled-indices = [ + available-indices = [ "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "collections" diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 8dac4249..3dda19cb 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -1,4 +1,5 @@ [config] + disabled-indices = [] [config.web-socket] server-url = "localhost:22111" data-marshaller-type = "json" diff --git a/config/config.go b/config/config.go index ae81d541..b726953f 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,7 @@ package config // Config will hold the whole config file's data type Config struct { Config struct { - EnabledIndices []string `toml:"enabled-indices"` + AvailableIndices []string `toml:"available-indices"` AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` @@ -33,7 +33,8 @@ type Config struct { // ClusterConfig will hold the config for the Elasticsearch cluster type ClusterConfig struct { Config struct { - WebSocket struct { + DisabledIndices []string `toml:"disabled-indices"` + WebSocket struct { ServerURL string `toml:"server-url"` DataMarshallerType string `toml:"data-marshaller-type"` } `toml:"web-socket"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index f9c4198e..7cd7e043 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -63,10 +63,28 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin Url: clusterCfg.Config.ElasticCluster.URL, UserName: clusterCfg.Config.ElasticCluster.UserName, Password: clusterCfg.Config.ElasticCluster.Password, - EnabledIndexes: cfg.Config.EnabledIndices, + EnabledIndexes: prepareIndices(cfg.Config.AvailableIndices, clusterCfg.Config.DisabledIndices), Marshalizer: marshaller, Hasher: hasher, AddressPubkeyConverter: addressPubkeyConverter, ValidatorPubkeyConverter: validatorPubkeyConverter, }) } + +func prepareIndices(availableIndices, disabledIndices []string) []string { + indices := make([]string, 0) + + mapDisabledIndices := make(map[string]struct{}) + for _, index := range disabledIndices { + mapDisabledIndices[index] = struct{}{} + } + + for _, availableIndex := range availableIndices { + _, found := mapDisabledIndices[availableIndex] + if !found { + indices = append(indices, availableIndex) + } + } + + return indices +} diff --git a/factory/wsindexerFactory_test.go b/factory/wsindexerFactory_test.go new file mode 100644 index 00000000..c04a8816 --- /dev/null +++ b/factory/wsindexerFactory_test.go @@ -0,0 +1,23 @@ +package factory + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPrepareIndices(t *testing.T) { + t.Parallel() + + available := []string{"index1", "index2"} + disabled := []string{"index2", "index3"} + + res := prepareIndices(available, disabled) + require.Equal(t, []string{"index1"}, res) + + available = []string{"index1", "index2"} + disabled = []string{} + + res = prepareIndices(available, disabled) + require.Equal(t, []string{"index1", "index2"}, res) +} From 7d83c4025236d559fb09414a15afcf5dddee08af Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 15:07:21 +0300 Subject: [PATCH 63/96] latest elrond-go core --- go.mod | 2 +- go.sum | 4 ++-- process/wsindexer/unmarshal.go | 10 +++++----- process/wsindexer/unmarshal_test.go | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 93bb86dd..adc82d96 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 + github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0 github.com/ElrondNetwork/elrond-go-logger v1.0.8 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 467e258d..2d46d63a 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67 h1:ht6Poy66/oPd0sQjo20WQppRYY1aDkxV5YW1lIslWOs= -github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20220930105914-ea5b9503dc67/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0 h1:6KxArKihOCF1CaSa8XqHqoCfY2s2fnAugS+Sfp/N+t0= +github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= diff --git a/process/wsindexer/unmarshal.go b/process/wsindexer/unmarshal.go index 81a09eab..74be7b36 100644 --- a/process/wsindexer/unmarshal.go +++ b/process/wsindexer/unmarshal.go @@ -5,10 +5,10 @@ import ( "errors" "time" + "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" - outportData "github.com/ElrondNetwork/elrond-go-core/websocketOutportDriver/data" ) func (i *indexer) getArgsSaveBlock(marshaledData []byte) (*outport.ArgsSaveBlockData, error) { @@ -159,7 +159,7 @@ func (i *indexer) getBody(marshaledData []byte) (data.BodyHandler, error) { func (i *indexer) getHeader(marshaledData []byte) (data.HeaderHandler, error) { headerTypeStruct := struct { - HeaderType outportData.HeaderType + HeaderType core.HeaderType }{} err := i.marshaller.Unmarshal(&headerTypeStruct, marshaledData) @@ -168,19 +168,19 @@ func (i *indexer) getHeader(marshaledData []byte) (data.HeaderHandler, error) { } switch headerTypeStruct.HeaderType { - case outportData.MetaHeader: + case core.MetaHeader: hStruct := struct { H1 *block.Header `json:"Header"` }{} err = i.marshaller.Unmarshal(&hStruct, marshaledData) return hStruct.H1, err - case outportData.ShardHeaderV1: + case core.ShardHeaderV1: hStruct := struct { H1 *block.MetaBlock `json:"Header"` }{} err = i.marshaller.Unmarshal(&hStruct, marshaledData) return hStruct.H1, err - case outportData.ShardHeaderV2: + case core.ShardHeaderV2: hStruct := struct { H1 *block.HeaderV2 `json:"Header"` }{} diff --git a/process/wsindexer/unmarshal_test.go b/process/wsindexer/unmarshal_test.go index 1f7518ad..b52ea320 100644 --- a/process/wsindexer/unmarshal_test.go +++ b/process/wsindexer/unmarshal_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer" + "github.com/ElrondNetwork/elrond-go-core/core" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/outport" @@ -26,7 +27,7 @@ func TestGetHeaderAndBody(t *testing.T) { di, _ := NewIndexer(marshaller, nilDataIndexer) arg := &outportData.ArgsRevertIndexedBlock{ - HeaderType: outportData.ShardHeaderV2, + HeaderType: core.ShardHeaderV2, Header: &block.HeaderV2{ScheduledRootHash: []byte("aaaaaa")}, Body: &block.Body{MiniBlocks: []*block.MiniBlock{{}}}, } @@ -83,7 +84,7 @@ func TestGetPool(t *testing.T) { } argsSaveBlock := &outportData.ArgsSaveBlock{ - ArgsSaveBlockData: &outport.ArgsSaveBlockData{ + ArgsSaveBlockData: outport.ArgsSaveBlockData{ TransactionsPool: pool, }, } From fd94fbb64a8b240ad5c28f180648b5755277c18e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 15:25:34 +0300 Subject: [PATCH 64/96] small fix --- factory/wsIndexerFactory.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 7cd7e043..06bb8c13 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -80,10 +80,11 @@ func prepareIndices(availableIndices, disabledIndices []string) []string { } for _, availableIndex := range availableIndices { - _, found := mapDisabledIndices[availableIndex] - if !found { - indices = append(indices, availableIndex) + _, shouldSkip := mapDisabledIndices[availableIndex] + if shouldSkip { + continue } + indices = append(indices, availableIndex) } return indices From 104ae39604168754411b82787f815d515b13c7e4 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 15:40:50 +0300 Subject: [PATCH 65/96] change action --- .github/workflows/golangci-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 989a0015..ac7d9c40 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -1,7 +1,7 @@ name: golangci-lint on: push: - branches: [ master, development, feat/*, rc/* ] + branches: [ master ] pull_request: branches: [ master, development, feat/*, rc/* ] From e1b24133ebc77ae652e26d3375defd0c2b1f6a53 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 16:12:27 +0300 Subject: [PATCH 66/96] fixes after merge --- go.mod | 3 --- process/accounts/accountsProcessor_test.go | 0 process/elasticproc/logsevents/fungibleESDTProcessor_test.go | 4 +++- process/elasticproc/logsevents/nftsProcessor_test.go | 1 - 4 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 process/accounts/accountsProcessor_test.go diff --git a/go.mod b/go.mod index 7e524af4..adc82d96 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,6 @@ require ( github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0 github.com/ElrondNetwork/elrond-go-logger v1.0.8 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 - github.com/ElrondNetwork/elrond-go-core v1.1.20 - github.com/ElrondNetwork/elrond-go-logger v1.0.7 - github.com/ElrondNetwork/elrond-vm-common v1.3.7 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 github.com/stretchr/testify v1.7.0 diff --git a/process/accounts/accountsProcessor_test.go b/process/accounts/accountsProcessor_test.go deleted file mode 100644 index e69de29b..00000000 diff --git a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go index 7a16ef14..56c66161 100644 --- a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go +++ b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go @@ -119,7 +119,7 @@ func TestProcessLogsAndEventsESDT_CrossShardOnDestination(t *testing.T) { func TestNftsProcessor_processLogAndEventsFungibleESDT_Wipe(t *testing.T) { t.Parallel() - nftsProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{}) + nftsProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) events := &transaction.Event{ Address: []byte("addr"), @@ -134,6 +134,8 @@ func TestNftsProcessor_processLogAndEventsFungibleESDT_Wipe(t *testing.T) { accounts: altered, timestamp: 10000, tokensSupply: data.NewTokensInfo(), + selfShardID: 2, + numOfShards: 3, }) require.Equal(t, "esdt-0123", res.identifier) require.Equal(t, "0", res.value) diff --git a/process/elasticproc/logsevents/nftsProcessor_test.go b/process/elasticproc/logsevents/nftsProcessor_test.go index a79f8236..5812a676 100644 --- a/process/elasticproc/logsevents/nftsProcessor_test.go +++ b/process/elasticproc/logsevents/nftsProcessor_test.go @@ -133,7 +133,6 @@ func TestNftsProcessor_processLogAndEventsNFTs_Wipe(t *testing.T) { accounts: altered, timestamp: 10000, tokensSupply: tokensSupply, - tokensSupply: data.NewTokensInfo(), numOfShards: 3, selfShardID: 2, }) From 91059d1000b1afefc25b15848eb6a3de9b85395a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 18:42:46 +0300 Subject: [PATCH 67/96] push --- scripts/observers/.env | 7 +++ scripts/observers/clean.py | 13 +++++ scripts/observers/config.py | 110 ++++++++++++++++++++++++++++++++++++ scripts/observers/start.py | 33 +++++++++++ scripts/observers/stop.py | 3 + 5 files changed, 166 insertions(+) create mode 100644 scripts/observers/.env create mode 100644 scripts/observers/clean.py create mode 100644 scripts/observers/config.py create mode 100644 scripts/observers/start.py create mode 100644 scripts/observers/stop.py diff --git a/scripts/observers/.env b/scripts/observers/.env new file mode 100644 index 00000000..229b7db3 --- /dev/null +++ b/scripts/observers/.env @@ -0,0 +1,7 @@ +CHAIN_ID="T" +ELROND_CONFIG_URL="https://github.com/ElrondNetwork/elrond-config-testnet" +ELROND_CONFIG_BRANCH="extend-config-toml-for-t.1.40" +ELROND_GO_URL="https://github.com/ElrondNetwork/elrond-go" +ELROND_GO_BRANCH="feat/altered-accounts" +WORKING_DIRECTORY="/IndexerObservers/" +OBSERVER_DIR="observer_shard_" diff --git a/scripts/observers/clean.py b/scripts/observers/clean.py new file mode 100644 index 00000000..7b3b7429 --- /dev/null +++ b/scripts/observers/clean.py @@ -0,0 +1,13 @@ +import shutil +import os +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + +working_dir = str(Path.home()) + str(os.getenv('WORKING_DIRECTORY')) + +try: + shutil.rmtree(working_dir) +except: + print('did nothing') diff --git a/scripts/observers/config.py b/scripts/observers/config.py new file mode 100644 index 00000000..f1039650 --- /dev/null +++ b/scripts/observers/config.py @@ -0,0 +1,110 @@ +import os +from pathlib import Path +from git import Repo +import shutil +from dotenv import load_dotenv +import toml + +load_dotenv() + +observer_dir = str(os.getenv('OBSERVER_DIR')) +METACHAIN = 4294967295 + + +def update_toml_indexer(path, shard_id): + # prefs.toml + data1 = toml.load(path+"/prefs.toml") + data1['config']['web-socket']['server-url'] = str(shard_id) + if shard_id != METACHAIN: + data1['config']['web-socket']['server-url'] = "localhost:" + str(22111+shard_id) + else: + data1['config']['web-socket']['server-url'] = "localhost:" + str(22111+10) + f = open(path+"/prefs.toml", 'w') + toml.dump(data1, f) + f.close() + + +def update_toml_node(path, shard_id): + # prefs.toml + data1 = toml.load(path+"/prefs.toml") + data1['Preferences']['DestinationShardAsObserver'] = str(shard_id) + f = open(path+"/prefs.toml", 'w') + toml.dump(data1, f) + f.close() + + # external.toml + data2 = toml.load(path+"/external.toml") + data2['WebSocketConnector']['Enabled'] = True + if shard_id != METACHAIN: + data2['WebSocketConnector']['URL'] = "localhost:" + str(22111+shard_id) + else: + data2['WebSocketConnector']['URL'] = "localhost:" + str(22111+10) + f = open(path+"/external.toml", 'w') + toml.dump(data2, f) + f.close() + + +def prepare_observer(shard_id): + current_observer = observer_dir + str(shard_id) + working_dir_observer = working_dir + current_observer + os.mkdir(working_dir_observer) + os.mkdir(working_dir_observer + "/indexer") + os.mkdir(working_dir_observer + "/node") + + node_config = working_dir_observer + "/node/config" + indexer_config = working_dir_observer + "/indexer/config" + + shutil.copytree(config_folder, node_config) + shutil.copytree("../../cmd/elasticindexer/config", indexer_config) + shutil.copyfile("../../cmd/elasticindexer/elasticindexer", working_dir_observer+"/indexer/elasticindexer") + os.system("chmod +x " + working_dir_observer+"/indexer/elasticindexer") + shutil.copyfile(working_dir + "/elrond-go/cmd/node/node", working_dir_observer+"/node/node") + os.system("chmod +x " + working_dir_observer+"/node/node") + update_toml_node(node_config, shard_id) + update_toml_indexer(indexer_config, shard_id) + + +working_dir = str(Path.home()) + str(os.getenv('WORKING_DIRECTORY')) +try: + os.makedirs(working_dir) +except FileExistsError: + print("working directory already exits") + +# CLONE elrond-config +print("cloning elrond-config....") +config_folder = working_dir + "/config" +if not os.path.isdir(config_folder): + Repo.clone_from(os.getenv('ELROND_CONFIG_URL'), config_folder) + +repo_cfg = Repo(config_folder) +repo_cfg.git.checkout(os.getenv('ELROND_CONFIG_BRANCH')) + +# CLONE elrond-go +print("cloning elrond-go....") +elrond_go_folder = working_dir + "/elrond-go" +if not os.path.isdir(elrond_go_folder): + Repo.clone_from(os.getenv('ELROND_GO_URL'), elrond_go_folder) + +repo_elrond_go = Repo(elrond_go_folder) +repo_elrond_go.git.checkout(os.getenv('ELROND_GO_BRANCH')) + +# build binary elrond-go +print("building node...") +current_dir = os.getcwd() +os.chdir(elrond_go_folder + "/cmd/node") +os.system("go build") +os.chdir(current_dir) + +# build binary indexer +print("building indexer...") +os.chdir("../../cmd/elasticindexer") +os.system("go build") + +os.chdir(current_dir) + +# prepare observers +print("preparing config...") +prepare_observer(0) +prepare_observer(1) +prepare_observer(2) +prepare_observer(4294967295) diff --git a/scripts/observers/start.py b/scripts/observers/start.py new file mode 100644 index 00000000..9cc235be --- /dev/null +++ b/scripts/observers/start.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + +working_dir = str(Path.home()) + str(os.getenv('WORKING_DIRECTORY')) +observer_dir = str(os.getenv('OBSERVER_DIR')) + + +def start_observer(shard_id): + current_observer = observer_dir + str(shard_id) + working_dir_observer = working_dir + current_observer + + current_directory = os.getcwd() + # start observer + os.chdir(working_dir_observer + "/node") + command = "./node" + " --log-level *:DEBUG --no-key --log-save" + # os.system("gnome-terminal 'bash -c \"" + command + ";bash\"'") + os.system("screen -d -m -S obs" + str(shard_id) + " " + command) + os.chdir(current_directory) + # start indexer + os.chdir(working_dir_observer + "/indexer") + command = "./elasticindexer" + " --log-level *:DEBUG --log-save" + # os.system("gnome-terminal 'bash -c \"" + command + ";bash\"'") + os.system("screen -d -m -S indexer" + str(shard_id) + " " + command) + os.chdir(current_directory) + + +start_observer(0) +start_observer(1) +start_observer(2) +start_observer(4294967295) diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py new file mode 100644 index 00000000..5198940f --- /dev/null +++ b/scripts/observers/stop.py @@ -0,0 +1,3 @@ +import os + +os.system("killall screen") From a1edbfac61f7df0938d7da8d1c23efec7406eaff Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 19:07:17 +0300 Subject: [PATCH 68/96] extra checks and prints --- scripts/observers/.env | 3 ++- scripts/observers/clean.py | 1 + scripts/observers/config.py | 7 +++++-- scripts/observers/start.py | 8 ++++++++ scripts/observers/stop.py | 1 + 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/observers/.env b/scripts/observers/.env index 229b7db3..4d17bcfd 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -1,7 +1,8 @@ -CHAIN_ID="T" ELROND_CONFIG_URL="https://github.com/ElrondNetwork/elrond-config-testnet" ELROND_CONFIG_BRANCH="extend-config-toml-for-t.1.40" + ELROND_GO_URL="https://github.com/ElrondNetwork/elrond-go" ELROND_GO_BRANCH="feat/altered-accounts" + WORKING_DIRECTORY="/IndexerObservers/" OBSERVER_DIR="observer_shard_" diff --git a/scripts/observers/clean.py b/scripts/observers/clean.py index 7b3b7429..9dcd567a 100644 --- a/scripts/observers/clean.py +++ b/scripts/observers/clean.py @@ -9,5 +9,6 @@ try: shutil.rmtree(working_dir) + print("done") except: print('did nothing') diff --git a/scripts/observers/config.py b/scripts/observers/config.py index f1039650..03c3acea 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -1,9 +1,10 @@ import os +import toml +import sys +import shutil from pathlib import Path from git import Repo -import shutil from dotenv import load_dotenv -import toml load_dotenv() @@ -69,6 +70,8 @@ def prepare_observer(shard_id): os.makedirs(working_dir) except FileExistsError: print("working directory already exits") + print("use `python3 clean.py` command first") + sys.exit() # CLONE elrond-config print("cloning elrond-config....") diff --git a/scripts/observers/start.py b/scripts/observers/start.py index 9cc235be..e9464532 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -1,4 +1,5 @@ import os +import sys from pathlib import Path from dotenv import load_dotenv @@ -27,7 +28,14 @@ def start_observer(shard_id): os.chdir(current_directory) +if not os.path.exists(working_dir): + print("working directory folder is missing...you should run first `python3 config.py` command") + sys.exit() + +print("staring observers and indexers....") + start_observer(0) start_observer(1) start_observer(2) start_observer(4294967295) +print("done") diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py index 5198940f..a62f905b 100644 --- a/scripts/observers/stop.py +++ b/scripts/observers/stop.py @@ -1,3 +1,4 @@ import os os.system("killall screen") +print("done") From 63e3991db9f227f4599fc3e3a8af91f781e35f22 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Oct 2022 19:07:48 +0300 Subject: [PATCH 69/96] small fix --- scripts/observers/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 03c3acea..3f738623 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -69,7 +69,7 @@ def prepare_observer(shard_id): try: os.makedirs(working_dir) except FileExistsError: - print("working directory already exits") + print("working directory already exists") print("use `python3 clean.py` command first") sys.exit() From 88fea1f95864f8f009af4432b852ccc9bb8a5701 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 25 Oct 2022 10:56:30 +0300 Subject: [PATCH 70/96] remove altered accounts part --- data/transaction.go | 33 ---- go.mod | 2 +- go.sum | 6 +- mock/dbAccountsHandlerStub.go | 2 +- .../elasticproc/accounts/accountsProcessor.go | 51 ++---- .../accounts/accountsProcessor_test.go | 37 ++-- process/elasticproc/elasticProcessor.go | 5 +- process/elasticproc/elasticProcessor_test.go | 4 +- process/elasticproc/interface.go | 2 +- .../logsevents/fungibleESDTProcessor.go | 108 ------------ .../logsevents/fungibleESDTProcessor_test.go | 157 ----------------- process/elasticproc/logsevents/interface.go | 13 +- .../logsevents/logsAndEventsProcessor.go | 16 +- .../logsevents/logsAndEventsProcessor_test.go | 4 - process/elasticproc/logsevents/logsData.go | 3 - .../elasticproc/logsevents/nftsProcessor.go | 51 ++---- .../logsevents/nftsProcessor_test.go | 79 --------- .../logsevents/nftsPropertiesProcessor.go | 1 - .../smartContractResultsProcessor.go | 26 --- .../smartContractResultsProcessor_test.go | 22 --- .../transactions/transactionDBBuilder.go | 61 ++++--- .../transactions/transactionDBBuilder_test.go | 47 +++-- .../transactions/transactionsGrouper.go | 33 ---- .../transactions/transactionsGrouper_test.go | 13 +- .../transactions/transactionsProcessor.go | 11 +- .../transactionsProcessor_test.go | 161 ------------------ 26 files changed, 120 insertions(+), 828 deletions(-) delete mode 100644 process/elasticproc/logsevents/fungibleESDTProcessor.go delete mode 100644 process/elasticproc/logsevents/fungibleESDTProcessor_test.go diff --git a/data/transaction.go b/data/transaction.go index 05b93a65..d06665ca 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -1,7 +1,6 @@ package data import ( - "math/big" "time" ) @@ -42,42 +41,11 @@ type Transaction struct { IsRelayed bool `json:"isRelayed,omitempty"` Version uint32 `json:"version,omitempty"` SmartContractResults []*ScResult `json:"-"` - ReceiverAddressBytes []byte `json:"-"` Hash string `json:"-"` BlockHash string `json:"-"` HadRefund bool `json:"-"` } -// GetGasLimit will return transaction gas limit -func (t *Transaction) GetGasLimit() uint64 { - return t.GasLimit -} - -// GetGasPrice will return transaction gas price -func (t *Transaction) GetGasPrice() uint64 { - return t.GasPrice -} - -// GetData will return transaction data field -func (t *Transaction) GetData() []byte { - return t.Data -} - -// GetRcvAddr will return transaction receiver address -func (t *Transaction) GetRcvAddr() []byte { - return t.ReceiverAddressBytes -} - -// GetValue wil return transaction value -func (t *Transaction) GetValue() *big.Int { - bigIntValue, ok := big.NewInt(0).SetString(t.Value, 10) - if !ok { - return big.NewInt(0) - } - - return bigIntValue -} - // Receipt is a structure containing all the fields that need to be safe for a Receipt type Receipt struct { Hash string `json:"-"` @@ -93,7 +61,6 @@ type PreparedResults struct { Transactions []*Transaction ScResults []*ScResult Receipts []*Receipt - AlteredAccts AlteredAccountsHandler TxHashStatus map[string]string TxHashFee map[string]*FeeData } diff --git a/go.mod b/go.mod index adc82d96..97de075f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0 + github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024142346-a31256ced50b github.com/ElrondNetwork/elrond-go-logger v1.0.8 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 2d46d63a..6c08bcd7 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0 h1:6KxArKihOCF1CaSa8XqHqoCfY2s2fnAugS+Sfp/N+t0= -github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024141157-ab57db14b806 h1:/IZ0Z3DK5xpfbAxGrcOpmF4UwQ8NhXBzUoa9YZBGrE8= +github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024141157-ab57db14b806/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024142346-a31256ced50b h1:xCVyWzdljqb/477iNtdAon7Ixh/2H6oB5ZetQGBNas4= +github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024142346-a31256ced50b/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= diff --git a/mock/dbAccountsHandlerStub.go b/mock/dbAccountsHandlerStub.go index b44070bf..37240df5 100644 --- a/mock/dbAccountsHandlerStub.go +++ b/mock/dbAccountsHandlerStub.go @@ -12,7 +12,7 @@ type DBAccountsHandlerStub struct { } // GetAccounts - -func (dba *DBAccountsHandlerStub) GetAccounts(_ data.AlteredAccountsHandler, _ map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { +func (dba *DBAccountsHandlerStub) GetAccounts(_ map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { return nil, nil } diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index 2d19e3de..7808185a 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -42,25 +42,13 @@ func NewAccountsProcessor( }, nil } -// TODO: refactor this as the altered accounts are already computed on the node. EN-12389 // GetAccounts will get accounts for regular operations and esdt operations -func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { +func (ap *accountsProcessor) GetAccounts(coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) - if check.IfNil(alteredAccounts) { - return regularAccountsToIndex, accountsToIndexESDT - } - - allAlteredAccounts := alteredAccounts.GetAll() - for address, altered := range allAlteredAccounts { - alteredAccount := coreAlteredAccounts[address] - if alteredAccount == nil { - log.Warn("account not found in core altered accounts map", "address", address) - continue - } - - regularAccounts, esdtAccounts := splitAlteredAccounts(alteredAccount, altered) + for _, alteredAccount := range coreAlteredAccounts { + regularAccounts, esdtAccounts := splitAlteredAccounts(alteredAccount) regularAccountsToIndex = append(regularAccountsToIndex, regularAccounts...) accountsToIndexESDT = append(accountsToIndexESDT, esdtAccounts...) @@ -71,31 +59,26 @@ func (ap *accountsProcessor) GetAccounts(alteredAccounts data.AlteredAccountsHan func splitAlteredAccounts( account *outport.AlteredAccount, - altered []*data.AlteredAccount, ) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) - for _, info := range altered { - if info.IsESDTOperation || info.IsNFTOperation { - accountsToIndexESDT = append(accountsToIndexESDT, &data.AccountESDT{ - Account: account, - TokenIdentifier: info.TokenIdentifier, - IsSender: info.IsSender, - IsNFTOperation: info.IsNFTOperation, - NFTNonce: info.NFTNonce, - IsNFTCreate: info.IsNFTCreate, - }) - } - - // if the balance of the ESDT receiver is 0 the receiver is a new account most probably, and we should index it - ignoreReceiver := !info.BalanceChange && notZeroBalance(account.Balance) && !info.IsSender - if ignoreReceiver { - continue - } + //if the balance of the ESDT receiver is 0 the receiver is a new account most probably, and we should index it + ignoreAddress := !account.BalanceChange && notZeroBalance(account.Balance) && !account.IsSender + if !ignoreAddress { regularAccountsToIndex = append(regularAccountsToIndex, &data.Account{ UserAccount: account, - IsSender: info.IsSender, + IsSender: account.IsSender, + }) + } + + for _, info := range account.Tokens { + accountsToIndexESDT = append(accountsToIndexESDT, &data.AccountESDT{ + Account: account, + TokenIdentifier: info.Identifier, + IsSender: account.IsSender, + NFTNonce: info.Nonce, + IsNFTCreate: info.IsNFTCreate, }) } diff --git a/process/elasticproc/accounts/accountsProcessor_test.go b/process/elasticproc/accounts/accountsProcessor_test.go index 92f9b732..bab917c7 100644 --- a/process/elasticproc/accounts/accountsProcessor_test.go +++ b/process/elasticproc/accounts/accountsProcessor_test.go @@ -65,7 +65,7 @@ func TestAccountsProcessor_GetAccountsWithNil(t *testing.T) { ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) - regularAccounts, esdtAccounts := ap.GetAccounts(nil, nil) + regularAccounts, esdtAccounts := ap.GetAccounts(nil) require.Len(t, regularAccounts, 0) require.Len(t, esdtAccounts, 0) } @@ -195,13 +195,7 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) - alteredAccounts := data.NewAlteredAccounts() - alteredAccounts.Add(addr, &data.AlteredAccount{ - IsESDTOperation: false, - TokenIdentifier: "", - }) - - accounts, esdtAccounts := ap.GetAccounts(alteredAccounts, alteredAccountsMap) + accounts, esdtAccounts := ap.GetAccounts(alteredAccountsMap) require.Equal(t, 0, len(esdtAccounts)) require.Equal(t, []*data.Account{ { @@ -217,6 +211,11 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { acc := &outport.AlteredAccount{ Address: addr, Balance: "1", + Tokens: []*outport.AccountTokenData{ + { + Identifier: "token", + }, + }, } alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, @@ -224,12 +223,7 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) - alteredAccounts := data.NewAlteredAccounts() - alteredAccounts.Add(addr, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: "token", - }) - accounts, esdtAccounts := ap.GetAccounts(alteredAccounts, alteredAccountsMap) + accounts, esdtAccounts := ap.GetAccounts(alteredAccountsMap) require.Equal(t, 0, len(accounts)) require.Equal(t, []*data.AccountESDT{ {Account: acc, TokenIdentifier: "token"}, @@ -240,19 +234,20 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco t.Parallel() addr := "aaaabbbb" - acc := &outport.AlteredAccount{} + acc := &outport.AlteredAccount{ + Tokens: []*outport.AccountTokenData{ + { + Identifier: "token", + }, + }, + } alteredAccountsMap := map[string]*outport.AlteredAccount{ addr: acc, } ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) require.NotNil(t, ap) - alteredAccounts := data.NewAlteredAccounts() - alteredAccounts.Add(addr, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: "token", - }) - accounts, esdtAccounts := ap.GetAccounts(alteredAccounts, alteredAccountsMap) + accounts, esdtAccounts := ap.GetAccounts(alteredAccountsMap) require.Equal(t, 1, len(accounts)) require.Equal(t, []*data.AccountESDT{ {Account: acc, TokenIdentifier: "token"}, diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index b75044f1..1e232039 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -438,7 +438,7 @@ func (ei *elasticProcessor) SaveTransactions( } tagsCount := tags.NewTagsCount() - err = ei.indexAlteredAccounts(headerTimestamp, preparedResults.AlteredAccts, logsData.NFTsDataUpdates, coreAlteredAccounts, buffers, tagsCount, header.GetShardID()) + err = ei.indexAlteredAccounts(headerTimestamp, logsData.NFTsDataUpdates, coreAlteredAccounts, buffers, tagsCount, header.GetShardID()) if err != nil { return err } @@ -601,14 +601,13 @@ func (ei *elasticProcessor) SaveRoundsInfo(info []*data.RoundInfo) error { func (ei *elasticProcessor) indexAlteredAccounts( timestamp uint64, - alteredAccounts data.AlteredAccountsHandler, updatesNFTsData []*data.NFTDataUpdate, coreAlteredAccounts map[string]*outport.AlteredAccount, buffSlice *data.BufferSlice, tagsCount data.CountTags, shardID uint32, ) error { - regularAccountsToIndex, accountsToIndexESDT := ei.accountsProc.GetAccounts(alteredAccounts, coreAlteredAccounts) + regularAccountsToIndex, accountsToIndexESDT := ei.accountsProc.GetAccounts(coreAlteredAccounts) err := ei.saveAccounts(timestamp, regularAccountsToIndex, buffSlice, shardID) if err != nil { diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 9a2d4995..9dacd3f3 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -637,7 +637,6 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) Transactions: nil, ScResults: nil, Receipts: nil, - AlteredAccts: nil, } }, SerializeScResultsCalled: func(scrs []*data.ScResult, _ *data.BufferSlice, _ string) error { @@ -681,9 +680,8 @@ func TestElasticProcessor_IndexAlteredAccounts(t *testing.T) { elasticSearchProc.enabledIndexes[dataindexer.AccountsESDTHistoryIndex] = struct{}{} buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - alteredAccounts := data.NewAlteredAccounts() tagsCount := tags.NewTagsCount() - err := elasticSearchProc.indexAlteredAccounts(100, alteredAccounts, nil, nil, buffSlice, tagsCount, 0) + err := elasticSearchProc.indexAlteredAccounts(100, nil, nil, buffSlice, tagsCount, 0) require.Nil(t, err) require.True(t, called) } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index fbaea63c..17eecd1d 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -30,7 +30,7 @@ type DatabaseClientHandler interface { // DBAccountHandler defines the actions that an accounts' handler should do type DBAccountHandler interface { - GetAccounts(alteredAccounts data.AlteredAccountsHandler, coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) + GetAccounts(coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) PrepareRegularAccountsMap(timestamp uint64, accounts []*data.Account, shardID uint32) map[string]*data.AccountInfo PrepareAccountsMapESDT(timestamp uint64, accounts []*data.AccountESDT, tagsCount data.CountTags, shardID uint32) (map[string]*data.AccountInfo, data.TokensHandler) PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo, shardID uint32) map[string]*data.AccountBalanceHistory diff --git a/process/elasticproc/logsevents/fungibleESDTProcessor.go b/process/elasticproc/logsevents/fungibleESDTProcessor.go deleted file mode 100644 index 6745ec62..00000000 --- a/process/elasticproc/logsevents/fungibleESDTProcessor.go +++ /dev/null @@ -1,108 +0,0 @@ -package logsevents - -import ( - "math/big" - - "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elrond-go-core/core" - "github.com/ElrondNetwork/elrond-go-core/core/sharding" - coreData "github.com/ElrondNetwork/elrond-go-core/data" -) - -const ( - numTopicsWithReceiverAddress = 4 -) - -type fungibleESDTProcessor struct { - pubKeyConverter core.PubkeyConverter - fungibleOperationsIdentifiers map[string]struct{} -} - -func newFungibleESDTProcessor(pubKeyConverter core.PubkeyConverter) *fungibleESDTProcessor { - return &fungibleESDTProcessor{ - pubKeyConverter: pubKeyConverter, - fungibleOperationsIdentifiers: map[string]struct{}{ - core.BuiltInFunctionESDTTransfer: {}, - core.BuiltInFunctionESDTBurn: {}, - core.BuiltInFunctionESDTLocalMint: {}, - core.BuiltInFunctionESDTLocalBurn: {}, - core.BuiltInFunctionESDTWipe: {}, - core.BuiltInFunctionMultiESDTNFTTransfer: {}, - }, - } -} - -func (fep *fungibleESDTProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { - identifier := args.event.GetIdentifier() - _, ok := fep.fungibleOperationsIdentifiers[string(identifier)] - if !ok { - return argOutputProcessEvent{} - } - - topics := args.event.GetTopics() - nonceBig := big.NewInt(0).SetBytes(topics[1]) - if nonceBig.Uint64() > 0 { - // this is a semi-fungible token so we should return - return argOutputProcessEvent{} - } - - address := args.event.GetAddress() - if len(topics) < numTopicsWithReceiverAddress-1 { - return argOutputProcessEvent{ - processed: true, - } - } - - selfShardID := args.selfShardID - senderShardID := sharding.ComputeShardID(address, args.numOfShards) - if senderShardID == selfShardID { - fep.processEventOnSenderShard(args.event, args.accounts) - } - - tokenID, valueStr, receiver, receiverShardID := fep.processEventDestination(args, selfShardID) - return argOutputProcessEvent{ - identifier: tokenID, - value: valueStr, - processed: true, - receiver: receiver, - receiverShardID: receiverShardID, - } -} - -func (fep *fungibleESDTProcessor) processEventOnSenderShard(event coreData.EventHandler, accounts data.AlteredAccountsHandler) { - topics := event.GetTopics() - tokenID := topics[0] - - encodedAddr := fep.pubKeyConverter.Encode(event.GetAddress()) - accounts.Add(encodedAddr, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: string(tokenID), - }) -} - -func (fep *fungibleESDTProcessor) processEventDestination( - args *argsProcessEvent, - selfShardID uint32, -) (string, string, string, uint32) { - topics := args.event.GetTopics() - tokenID := string(topics[0]) - valueBig := big.NewInt(0).SetBytes(topics[2]) - - if len(topics) < numTopicsWithReceiverAddress { - return tokenID, valueBig.String(), "", 0 - } - - receiverAddr := topics[3] - receiverShardID := sharding.ComputeShardID(receiverAddr, args.numOfShards) - encodedReceiver := fep.pubKeyConverter.Encode(receiverAddr) - if receiverShardID != selfShardID { - return tokenID, valueBig.String(), "", 0 - } - - args.accounts.Add(encodedReceiver, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: tokenID, - }) - - return tokenID, valueBig.String(), encodedReceiver, receiverShardID -} diff --git a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go b/process/elasticproc/logsevents/fungibleESDTProcessor_test.go deleted file mode 100644 index 56c66161..00000000 --- a/process/elasticproc/logsevents/fungibleESDTProcessor_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package logsevents - -import ( - "math/big" - "testing" - - "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elrond-go-core/core" - "github.com/ElrondNetwork/elrond-go-core/data/transaction" - "github.com/stretchr/testify/require" -) - -func TestProcessLogsAndEventsESDT_IntraShard(t *testing.T) { - t.Parallel() - - fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) - - event := &transaction.Event{ - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).Bytes(), big.NewInt(0).SetUint64(100).Bytes(), []byte("receiver")}, - } - altered := data.NewAlteredAccounts() - - fungibleProc.processEvent(&argsProcessEvent{ - event: event, - accounts: altered, - selfShardID: 2, - numOfShards: 3, - }) - - alteredAddrSender, ok := altered.Get("61646472") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: "my-token", - }, alteredAddrSender[0]) - - alteredAddrReceiver, ok := altered.Get("7265636569766572") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: "my-token", - }, alteredAddrReceiver[0]) -} - -func TestProcessLogsAndEventsESDT_CrossShardOnSource(t *testing.T) { - t.Parallel() - - receiverAddr := []byte("a") - fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) - - event := &transaction.Event{ - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).Bytes(), big.NewInt(0).SetUint64(100).Bytes(), receiverAddr}, - } - - altered := data.NewAlteredAccounts() - - res := fungibleProc.processEvent(&argsProcessEvent{ - event: event, - accounts: altered, - selfShardID: 2, - numOfShards: 3, - }) - require.Equal(t, "my-token", res.identifier) - require.Equal(t, "100", res.value) - require.Equal(t, true, res.processed) - - alteredAddrSender, ok := altered.Get("61646472") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: "my-token", - }, alteredAddrSender[0]) - - _, ok = altered.Get("61") - require.False(t, ok) -} - -func TestProcessLogsAndEventsESDT_CrossShardOnDestination(t *testing.T) { - t.Parallel() - - senderAddr := []byte("a") - receiverAddr := []byte("receiver") - fungibleProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) - - event := &transaction.Event{ - Address: senderAddr, - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).Bytes(), big.NewInt(0).SetUint64(100).Bytes(), receiverAddr}, - } - - altered := data.NewAlteredAccounts() - - res := fungibleProc.processEvent(&argsProcessEvent{ - event: event, - accounts: altered, - selfShardID: 2, - numOfShards: 3, - }) - require.Equal(t, "my-token", res.identifier) - require.Equal(t, "100", res.value) - require.Equal(t, true, res.processed) - - alteredAddrSender, ok := altered.Get("7265636569766572") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsESDTOperation: true, - TokenIdentifier: "my-token", - }, alteredAddrSender[0]) - - _, ok = altered.Get("61") - require.False(t, ok) -} - -func TestNftsProcessor_processLogAndEventsFungibleESDT_Wipe(t *testing.T) { - t.Parallel() - - nftsProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}) - - events := &transaction.Event{ - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTWipe), - Topics: [][]byte{[]byte("esdt-0123"), big.NewInt(0).SetUint64(0).Bytes(), big.NewInt(0).Bytes(), []byte("receiver")}, - } - - altered := data.NewAlteredAccounts() - - res := nftsProc.processEvent(&argsProcessEvent{ - event: events, - accounts: altered, - timestamp: 10000, - tokensSupply: data.NewTokensInfo(), - selfShardID: 2, - numOfShards: 3, - }) - require.Equal(t, "esdt-0123", res.identifier) - require.Equal(t, "0", res.value) - require.Equal(t, true, res.processed) - - alteredAddrSender, ok := altered.Get("61646472") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - TokenIdentifier: "esdt-0123", - IsESDTOperation: true, - }, alteredAddrSender[0]) - - alteredAddrReceiver, ok := altered.Get("7265636569766572") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - TokenIdentifier: "esdt-0123", - IsESDTOperation: true, - }, alteredAddrReceiver[0]) -} diff --git a/process/elasticproc/logsevents/interface.go b/process/elasticproc/logsevents/interface.go index 68a42814..20144c94 100644 --- a/process/elasticproc/logsevents/interface.go +++ b/process/elasticproc/logsevents/interface.go @@ -11,7 +11,6 @@ type argsProcessEvent struct { scDeploys map[string]*data.ScDeployInfo txs map[string]*data.Transaction event coreData.EventHandler - accounts data.AlteredAccountsHandler tokens data.TokensHandler tokensSupply data.TokensHandler tokenRolesAndProperties *tokeninfo.TokenRolesAndProperties @@ -22,14 +21,10 @@ type argsProcessEvent struct { } type argOutputProcessEvent struct { - identifier string - value string - receiver string - receiverShardID uint32 - tokenInfo *data.TokenInfo - delegator *data.Delegator - processed bool - updatePropNFT *data.NFTDataUpdate + tokenInfo *data.TokenInfo + delegator *data.Delegator + processed bool + updatePropNFT *data.NFTDataUpdate } type eventsProcessor interface { diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index e66a5b7e..5eead50a 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -64,7 +64,6 @@ func checkArgsLogsAndEventsProcessor(args ArgsLogsAndEventsProcessor) error { func createEventsProcessors(args ArgsLogsAndEventsProcessor) []eventsProcessor { nftsProc := newNFTsProcessor(args.PubKeyConverter, args.Marshalizer) - fungibleProc := newFungibleESDTProcessor(args.PubKeyConverter) scDeploysProc := newSCDeploysProcessor(args.PubKeyConverter) informativeProc := newInformativeLogsProcessor() updateNFTProc := newNFTsPropertiesProcessor(args.PubKeyConverter) @@ -73,14 +72,13 @@ func createEventsProcessors(args ArgsLogsAndEventsProcessor) []eventsProcessor { delegatorsProcessor := newDelegatorsProcessor(args.PubKeyConverter, args.BalanceConverter) eventsProcs := []eventsProcessor{ - fungibleProc, - nftsProc, scDeploysProc, informativeProc, updateNFTProc, esdtPropProc, esdtIssueProc, delegatorsProcessor, + nftsProc, } return eventsProcs @@ -94,7 +92,7 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( shardID uint32, numOfShards uint32, ) *data.PreparedLogsResults { - lep.logsData = newLogsData(timestamp, preparedResults.AlteredAccts, preparedResults.Transactions, preparedResults.ScResults) + lep.logsData = newLogsData(timestamp, preparedResults.Transactions, preparedResults.ScResults) for _, txLog := range logsAndEvents { if txLog == nil || check.IfNil(txLog.LogHandler) { @@ -133,7 +131,6 @@ func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byt event: event, txHashHexEncoded: logHashHexEncoded, logAddress: logAddress, - accounts: lep.logsData.accounts, tokens: lep.logsData.tokens, tokensSupply: lep.logsData.tokensSupply, timestamp: lep.logsData.timestamp, @@ -153,18 +150,13 @@ func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byt lep.logsData.nftsDataUpdates = append(lep.logsData.nftsDataUpdates, res.updatePropNFT) } - isEmptyIdentifier := res.identifier == "" - if isEmptyIdentifier && res.processed { - return - } - tx, ok := lep.logsData.txsMap[logHashHexEncoded] - if ok && !isEmptyIdentifier { + if ok { tx.HasOperations = true continue } scr, ok := lep.logsData.scrsMap[logHashHexEncoded] - if ok && !isEmptyIdentifier { + if ok { scr.HasOperations = true return } diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index ff4079cc..7a6f4fe6 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -130,7 +130,6 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }) } - altered := data.NewAlteredAccounts() res := &data.PreparedResults{ Transactions: []*data.Transaction{ { @@ -142,7 +141,6 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) Hash: "6832", }, }, - AlteredAccts: altered, } args := createMockArgs() @@ -262,7 +260,6 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsNFTBurn(t *testing.T) { TxHash: "h1", } - altered := data.NewAlteredAccounts() res := &data.PreparedResults{ Transactions: []*data.Transaction{ { @@ -274,7 +271,6 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsNFTBurn(t *testing.T) { Hash: "6832", }, }, - AlteredAccts: altered, } args := createMockArgs() diff --git a/process/elasticproc/logsevents/logsData.go b/process/elasticproc/logsevents/logsData.go index 11aaa163..9baf8c23 100644 --- a/process/elasticproc/logsevents/logsData.go +++ b/process/elasticproc/logsevents/logsData.go @@ -10,7 +10,6 @@ type logsData struct { timestamp uint64 tokens data.TokensHandler tokensSupply data.TokensHandler - accounts data.AlteredAccountsHandler txsMap map[string]*data.Transaction scrsMap map[string]*data.ScResult scDeploys map[string]*data.ScDeployInfo @@ -22,7 +21,6 @@ type logsData struct { func newLogsData( timestamp uint64, - accounts data.AlteredAccountsHandler, txs []*data.Transaction, scrs []*data.ScResult, ) *logsData { @@ -32,7 +30,6 @@ func newLogsData( ld.scrsMap = converters.ConvertScrsSliceIntoMap(scrs) ld.tokens = data.NewTokensInfo() ld.tokensSupply = data.NewTokensInfo() - ld.accounts = accounts ld.timestamp = timestamp ld.scDeploys = make(map[string]*data.ScDeployInfo) ld.tokensInfo = make([]*data.TokenInfo, 0) diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index 67e0a7a0..daec7189 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -14,6 +14,10 @@ import ( logger "github.com/ElrondNetwork/elrond-go-logger" ) +const ( + numTopicsWithReceiverAddress = 4 +) + var log = logger.GetOrCreate("indexer/process/logsevents") type nftsProcessor struct { @@ -30,12 +34,9 @@ func newNFTsProcessor( pubKeyConverter: pubKeyConverter, marshalizer: marshalizer, nftOperationsIdentifiers: map[string]struct{}{ - core.BuiltInFunctionESDTNFTTransfer: {}, - core.BuiltInFunctionESDTNFTBurn: {}, - core.BuiltInFunctionESDTNFTAddQuantity: {}, - core.BuiltInFunctionESDTNFTCreate: {}, - core.BuiltInFunctionMultiESDTNFTTransfer: {}, - core.BuiltInFunctionESDTWipe: {}, + core.BuiltInFunctionESDTNFTBurn: {}, + core.BuiltInFunctionESDTNFTCreate: {}, + core.BuiltInFunctionESDTWipe: {}, }, } } @@ -63,31 +64,23 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv sender := args.event.GetAddress() senderShardID := sharding.ComputeShardID(sender, args.numOfShards) if senderShardID == args.selfShardID { - np.processNFTEventOnSender(args.event, args.accounts, args.tokens, args.tokensSupply, args.timestamp) + np.processNFTEventOnSender(args.event, args.tokens, args.tokensSupply, args.timestamp) } token := string(topics[0]) identifier := converters.ComputeTokenIdentifier(token, nonceBig.Uint64()) - valueBig := big.NewInt(0).SetBytes(topics[2]) if !np.shouldAddReceiverData(args) { return argOutputProcessEvent{ - identifier: identifier, - value: valueBig.String(), - processed: true, + processed: true, } } receiver := args.event.GetTopics()[3] - encodedReceiver := np.pubKeyConverter.Encode(topics[3]) receiverShardID := sharding.ComputeShardID(receiver, args.numOfShards) if receiverShardID != args.selfShardID { return argOutputProcessEvent{ - identifier: identifier, - value: valueBig.String(), - processed: true, - receiver: encodedReceiver, - receiverShardID: receiverShardID, + processed: true, } } @@ -100,18 +93,8 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv }) } - args.accounts.Add(encodedReceiver, &data.AlteredAccount{ - IsNFTOperation: true, - TokenIdentifier: token, - NFTNonce: nonceBig.Uint64(), - }) - return argOutputProcessEvent{ - identifier: identifier, - value: valueBig.String(), - processed: true, - receiver: encodedReceiver, - receiverShardID: receiverShardID, + processed: true, } } @@ -129,17 +112,13 @@ func (np *nftsProcessor) shouldAddReceiverData(args *argsProcessEvent) bool { func (np *nftsProcessor) processNFTEventOnSender( event coreData.EventHandler, - accounts data.AlteredAccountsHandler, tokensCreateInfo data.TokensHandler, tokensSupply data.TokensHandler, timestamp uint64, ) { - sender := event.GetAddress() topics := event.GetTopics() token := string(topics[0]) nonceBig := big.NewInt(0).SetBytes(topics[1]) - bech32Addr := np.pubKeyConverter.Encode(sender) - eventIdentifier := string(event.GetIdentifier()) if eventIdentifier == core.BuiltInFunctionESDTNFTBurn || eventIdentifier == core.BuiltInFunctionESDTWipe { tokensSupply.Add(&data.TokenInfo{ @@ -151,14 +130,6 @@ func (np *nftsProcessor) processNFTEventOnSender( } isNFTCreate := eventIdentifier == core.BuiltInFunctionESDTNFTCreate - alteredAccount := &data.AlteredAccount{ - IsNFTOperation: true, - TokenIdentifier: token, - NFTNonce: nonceBig.Uint64(), - IsNFTCreate: isNFTCreate, - } - accounts.Add(bech32Addr, alteredAccount) - shouldReturn := !isNFTCreate || len(topics) < numTopicsWithReceiverAddress if shouldReturn { return diff --git a/process/elasticproc/logsevents/nftsProcessor_test.go b/process/elasticproc/logsevents/nftsProcessor_test.go index 5812a676..bf37adf4 100644 --- a/process/elasticproc/logsevents/nftsProcessor_test.go +++ b/process/elasticproc/logsevents/nftsProcessor_test.go @@ -34,30 +34,15 @@ func TestNftsProcessor_processLogAndEventsNFTs(t *testing.T) { nftsProc := newNFTsProcessor(&mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) - altered := data.NewAlteredAccounts() - tokensCreateInfo := data.NewTokensInfo() res := nftsProc.processEvent(&argsProcessEvent{ event: event, - accounts: altered, tokens: tokensCreateInfo, timestamp: 1000, selfShardID: 2, numOfShards: 3, }) - require.Equal(t, "my-token-13", res.identifier) - require.Equal(t, "1", res.value) require.Equal(t, true, res.processed) - - alteredAddr, ok := altered.Get("61646472") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsNFTOperation: true, - TokenIdentifier: "my-token", - NFTNonce: 19, - IsNFTCreate: true, - }, alteredAddr[0]) - require.Equal(t, &data.TokenInfo{ Identifier: "my-token-13", Token: "my-token", @@ -71,48 +56,6 @@ func TestNftsProcessor_processLogAndEventsNFTs(t *testing.T) { } -func TestNftsProcessor_processLogAndEventsNFTs_TransferNFT(t *testing.T) { - t.Parallel() - - nonce := uint64(19) - nftsProc := newNFTsProcessor(&mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}) - - events := &transaction.Event{ - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(nonce).Bytes(), big.NewInt(1).Bytes(), []byte("receiver")}, - } - - altered := data.NewAlteredAccounts() - - res := nftsProc.processEvent(&argsProcessEvent{ - event: events, - accounts: altered, - timestamp: 10000, - selfShardID: 2, - numOfShards: 3, - }) - require.Equal(t, "my-token-13", res.identifier) - require.Equal(t, "1", res.value) - require.Equal(t, true, res.processed) - - alteredAddrSender, ok := altered.Get("61646472") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsNFTOperation: true, - TokenIdentifier: "my-token", - NFTNonce: 19, - }, alteredAddrSender[0]) - - alteredAddrReceiver, ok := altered.Get("7265636569766572") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsNFTOperation: true, - TokenIdentifier: "my-token", - NFTNonce: 19, - }, alteredAddrReceiver[0]) -} - func TestNftsProcessor_processLogAndEventsNFTs_Wipe(t *testing.T) { t.Parallel() @@ -125,37 +68,15 @@ func TestNftsProcessor_processLogAndEventsNFTs_Wipe(t *testing.T) { Topics: [][]byte{[]byte("nft-0123"), big.NewInt(0).SetUint64(nonce).Bytes(), big.NewInt(1).Bytes(), []byte("receiver")}, } - altered := data.NewAlteredAccounts() - tokensSupply := data.NewTokensInfo() res := nftsProc.processEvent(&argsProcessEvent{ event: events, - accounts: altered, timestamp: 10000, tokensSupply: tokensSupply, numOfShards: 3, selfShardID: 2, }) - require.Equal(t, "nft-0123-14", res.identifier) - require.Equal(t, "1", res.value) require.Equal(t, true, res.processed) - - alteredAddrSender, ok := altered.Get("61646472") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsNFTOperation: true, - TokenIdentifier: "nft-0123", - NFTNonce: 20, - }, alteredAddrSender[0]) - - alteredAddrReceiver, ok := altered.Get("7265636569766572") - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - IsNFTOperation: true, - TokenIdentifier: "nft-0123", - NFTNonce: 20, - }, alteredAddrReceiver[0]) - require.Equal(t, &data.TokenInfo{ Identifier: "nft-0123-14", Token: "nft-0123", diff --git a/process/elasticproc/logsevents/nftsPropertiesProcessor.go b/process/elasticproc/logsevents/nftsPropertiesProcessor.go index 0de43341..eb001d9a 100644 --- a/process/elasticproc/logsevents/nftsPropertiesProcessor.go +++ b/process/elasticproc/logsevents/nftsPropertiesProcessor.go @@ -74,7 +74,6 @@ func (npp *nftsPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro return argOutputProcessEvent{ processed: true, - identifier: identifier, updatePropNFT: updateNFT, } } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index b6e962ad..2f3c7112 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -8,7 +8,6 @@ import ( indexerData "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elrond-go-core/core" - "github.com/ElrondNetwork/elrond-go-core/core/sharding" "github.com/ElrondNetwork/elrond-go-core/data" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -172,28 +171,3 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( InitialTxGasUsed: initialTxGasUsed, } } - -func (proc *smartContractResultsProcessor) addScrsReceiverToAlteredAccounts( - alteredAccounts indexerData.AlteredAccountsHandler, - scrs []*indexerData.ScResult, - selfShardID uint32, - numOfShards uint32, -) { - for _, scr := range scrs { - receiverAddr, _ := proc.pubKeyConverter.Decode(scr.Receiver) - if selfShardID != sharding.ComputeShardID(receiverAddr, numOfShards) { - continue - } - - balanceNotChanged := scr.Value == emptyString || scr.Value == "0" - if balanceNotChanged { - // the smart contract results that don't alter the balance of the receiver address should be ignored - continue - } - - alteredAccounts.Add(scr.Receiver, &indexerData.AlteredAccount{ - IsSender: false, - BalanceChange: true, - }) - } -} diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index becd2d7f..8a395050 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -71,25 +71,3 @@ func TestPrepareSmartContractResult(t *testing.T) { require.Equal(t, expectedTx, scRes) } - -func TestAddScrsReceiverToAlteredAccounts_ShouldWork(t *testing.T) { - t.Parallel() - - parser := createDataFieldParserMock() - scrsProc := newSmartContractResultsProcessor(&mock.PubkeyConverterMock{}, &mock.MarshalizerMock{}, &mock.HasherMock{}, parser) - - alteredAddress := data.NewAlteredAccounts() - scrs := []*data.ScResult{ - { - Sender: "010101", - Receiver: "020202", - Data: []byte("ESDTTransfer@544b4e2d626231323061@010f0cf064dd59200000"), - Value: "1", - }, - } - scrsProc.addScrsReceiverToAlteredAccounts(alteredAddress, scrs, 2, 3) - require.Equal(t, 1, alteredAddress.Len()) - - _, ok := alteredAddress.Get("020202") - require.True(t, ok) -} diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 884f27b7..7e9cbeb9 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -16,8 +16,6 @@ import ( datafield "github.com/ElrondNetwork/elrond-vm-common/parsers/dataField" ) -const emptyString = "" - type dbTransactionBuilder struct { addressPubkeyConverter core.PubkeyConverter dataFieldParser DataFieldParser @@ -49,36 +47,35 @@ func (dtb *dbTransactionBuilder) prepareTransaction( res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr, numOfShards) return &data.Transaction{ - Hash: hex.EncodeToString(txHash), - MBHash: hex.EncodeToString(mbHash), - Nonce: tx.Nonce, - Round: header.GetRound(), - Value: tx.Value.String(), - Receiver: dtb.addressPubkeyConverter.Encode(tx.RcvAddr), - Sender: dtb.addressPubkeyConverter.Encode(tx.SndAddr), - ReceiverShard: mb.ReceiverShardID, - SenderShard: mb.SenderShardID, - GasPrice: tx.GasPrice, - GasLimit: tx.GasLimit, - Data: tx.Data, - Signature: hex.EncodeToString(tx.Signature), - Timestamp: time.Duration(header.GetTimeStamp()), - Status: txStatus, - GasUsed: gasUsed, - InitialPaidFee: initialPaidFee.String(), - Fee: fee.String(), - ReceiverUserName: tx.RcvUserName, - SenderUserName: tx.SndUserName, - ReceiverAddressBytes: tx.RcvAddr, - IsScCall: isScCall, - Operation: res.Operation, - Function: res.Function, - ESDTValues: res.ESDTValues, - Tokens: res.Tokens, - Receivers: datafield.EncodeBytesSlice(dtb.addressPubkeyConverter.Encode, res.Receivers), - ReceiversShardIDs: res.ReceiversShardID, - IsRelayed: res.IsRelayed, - Version: tx.Version, + Hash: hex.EncodeToString(txHash), + MBHash: hex.EncodeToString(mbHash), + Nonce: tx.Nonce, + Round: header.GetRound(), + Value: tx.Value.String(), + Receiver: dtb.addressPubkeyConverter.Encode(tx.RcvAddr), + Sender: dtb.addressPubkeyConverter.Encode(tx.SndAddr), + ReceiverShard: mb.ReceiverShardID, + SenderShard: mb.SenderShardID, + GasPrice: tx.GasPrice, + GasLimit: tx.GasLimit, + Data: tx.Data, + Signature: hex.EncodeToString(tx.Signature), + Timestamp: time.Duration(header.GetTimeStamp()), + Status: txStatus, + GasUsed: gasUsed, + InitialPaidFee: initialPaidFee.String(), + Fee: fee.String(), + ReceiverUserName: tx.RcvUserName, + SenderUserName: tx.SndUserName, + IsScCall: isScCall, + Operation: res.Operation, + Function: res.Function, + ESDTValues: res.ESDTValues, + Tokens: res.Tokens, + Receivers: datafield.EncodeBytesSlice(dtb.addressPubkeyConverter.Encode, res.Receivers), + ReceiversShardIDs: res.ReceiversShardID, + IsRelayed: res.IsRelayed, + Version: tx.Version, } } diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index 951ec3c1..78448fba 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -51,30 +51,29 @@ func TestGetMoveBalanceTransaction(t *testing.T) { } expectedTx := &data.Transaction{ - Hash: hex.EncodeToString(txHash), - MBHash: hex.EncodeToString(mbHash), - Nonce: tx.Nonce, - Round: header.Round, - Value: tx.Value.String(), - Receiver: cp.addressPubkeyConverter.Encode(tx.RcvAddr), - Sender: cp.addressPubkeyConverter.Encode(tx.SndAddr), - ReceiverShard: mb.ReceiverShardID, - SenderShard: mb.SenderShardID, - GasPrice: gasPrice, - GasLimit: gasLimit, - GasUsed: uint64(500), - InitialPaidFee: "100", - Data: tx.Data, - Signature: hex.EncodeToString(tx.Signature), - Timestamp: time.Duration(header.GetTimeStamp()), - Status: status, - ReceiverAddressBytes: []byte("receiver"), - Fee: "100", - ReceiverUserName: []byte("rcv"), - SenderUserName: []byte("snd"), - Operation: "transfer", - Version: 1, - Receivers: []string{}, + Hash: hex.EncodeToString(txHash), + MBHash: hex.EncodeToString(mbHash), + Nonce: tx.Nonce, + Round: header.Round, + Value: tx.Value.String(), + Receiver: cp.addressPubkeyConverter.Encode(tx.RcvAddr), + Sender: cp.addressPubkeyConverter.Encode(tx.SndAddr), + ReceiverShard: mb.ReceiverShardID, + SenderShard: mb.SenderShardID, + GasPrice: gasPrice, + GasLimit: gasLimit, + GasUsed: uint64(500), + InitialPaidFee: "100", + Data: tx.Data, + Signature: hex.EncodeToString(tx.Signature), + Timestamp: time.Duration(header.GetTimeStamp()), + Status: status, + Fee: "100", + ReceiverUserName: []byte("rcv"), + SenderUserName: []byte("snd"), + Operation: "transfer", + Version: 1, + Receivers: []string{}, } dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500, big.NewInt(100), 3) diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index f2fe2d9e..74b361e4 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -39,7 +39,6 @@ func (tg *txsGrouper) groupNormalTxs( mb *block.MiniBlock, header coreData.HeaderHandler, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, - alteredAccounts data.AlteredAccountsHandler, isImportDB bool, numOfShards uint32, ) (map[string]*data.Transaction, error) { @@ -59,7 +58,6 @@ func (tg *txsGrouper) groupNormalTxs( continue } - tg.addToAlteredAddresses(dbTx, alteredAccounts, mb, selfShardID, false) if tg.shouldIndex(mb.ReceiverShardID, isImportDB, selfShardID) { transactions[string(txHash)] = dbTx } @@ -118,7 +116,6 @@ func (tg *txsGrouper) groupRewardsTxs( mb *block.MiniBlock, header coreData.HeaderHandler, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, - alteredAccounts data.AlteredAccountsHandler, isImportDB bool, ) (map[string]*data.Transaction, error) { rewardsTxs := make(map[string]*data.Transaction) @@ -136,7 +133,6 @@ func (tg *txsGrouper) groupRewardsTxs( continue } - tg.addToAlteredAddresses(rewardDBTx, alteredAccounts, mb, selfShardID, true) if tg.shouldIndex(mb.ReceiverShardID, isImportDB, selfShardID) { rewardsTxs[string(txHash)] = rewardDBTx } @@ -173,7 +169,6 @@ func (tg *txsGrouper) groupInvalidTxs( mb *block.MiniBlock, header coreData.HeaderHandler, txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, - alteredAccounts data.AlteredAccountsHandler, numOfShards uint32, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) @@ -189,7 +184,6 @@ func (tg *txsGrouper) groupInvalidTxs( continue } - tg.addToAlteredAddresses(invalidDBTx, alteredAccounts, mb, header.GetShardID(), false) transactions[string(txHash)] = invalidDBTx } @@ -258,30 +252,3 @@ func convertMapTxsToSlice(txs map[string]*data.Transaction) []*data.Transaction } return transactions } - -func (tg *txsGrouper) addToAlteredAddresses( - tx *data.Transaction, - alteredAccounts data.AlteredAccountsHandler, - miniBlock *block.MiniBlock, - selfShardID uint32, - isRewardTx bool, -) { - if selfShardID == miniBlock.SenderShardID && !isRewardTx { - alteredAccounts.Add(tx.Sender, &data.AlteredAccount{ - IsSender: true, - BalanceChange: true, - }) - } - - ignoreTransactionReceiver := tx.Status == transaction.TxStatusInvalid.String() || tx.Sender == tx.Receiver - if ignoreTransactionReceiver { - return - } - - if selfShardID == miniBlock.ReceiverShardID || miniBlock.ReceiverShardID == core.AllShardId { - alteredAccounts.Add(tx.Receiver, &data.AlteredAccount{ - IsSender: false, - BalanceChange: true, - }) - } -} diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index d03463e4..e40db4f9 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -4,7 +4,6 @@ import ( "math/big" "testing" - "github.com/ElrondNetwork/elastic-indexer-go/data" "github.com/ElrondNetwork/elastic-indexer-go/mock" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" @@ -39,11 +38,9 @@ func TestGroupNormalTxs(t *testing.T) { RcvAddr: []byte("receiver2"), }, 0, big.NewInt(0)), } - alteredAddresses := data.NewAlteredAccounts() - normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, alteredAddresses, false, 3) + normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, false, 3) require.Len(t, normalTxs, 2) - require.Equal(t, 4, alteredAddresses.Len()) } func TestGroupRewardsTxs(t *testing.T) { @@ -68,11 +65,9 @@ func TestGroupRewardsTxs(t *testing.T) { RcvAddr: []byte("receiver2"), }, 0, big.NewInt(0)), } - alteredAddresses := data.NewAlteredAccounts() - normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, alteredAddresses, false) + normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, false) require.Len(t, normalTxs, 2) - require.Equal(t, 2, alteredAddresses.Len()) } func TestGroupInvalidTxs(t *testing.T) { @@ -99,11 +94,9 @@ func TestGroupInvalidTxs(t *testing.T) { RcvAddr: []byte("receiver2"), }, 0, big.NewInt(0)), } - alteredAddresses := data.NewAlteredAccounts() - normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, alteredAddresses, 3) + normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, 3) require.Len(t, normalTxs, 2) - require.Equal(t, 2, alteredAddresses.Len()) } func TestGroupReceipts(t *testing.T) { diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index c325b6a5..d08a4060 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -77,11 +77,9 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( Transactions: []*data.Transaction{}, ScResults: []*data.ScResult{}, Receipts: []*data.Receipt{}, - AlteredAccts: data.NewAlteredAccounts(), } } - alteredAccounts := data.NewAlteredAccounts() normalTxs := make(map[string]*data.Transaction) rewardsTxs := make(map[string]*data.Transaction) @@ -92,21 +90,21 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( continue } - txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Txs, alteredAccounts, isImportDB, numOfShards) + txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Txs, isImportDB, numOfShards) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupNormalTxs", "error", errGroup) continue } mergeTxsMaps(normalTxs, txs) case block.RewardsBlock: - txs, errGroup := tdp.txsGrouper.groupRewardsTxs(mbIndex, mb, header, pool.Rewards, alteredAccounts, isImportDB) + txs, errGroup := tdp.txsGrouper.groupRewardsTxs(mbIndex, mb, header, pool.Rewards, isImportDB) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupRewardsTxs", "error", errGroup) continue } mergeTxsMaps(rewardsTxs, txs) case block.InvalidBlock: - txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, header, pool.Invalid, alteredAccounts, numOfShards) + txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, header, pool.Invalid, numOfShards) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupInvalidTxs", "error", errGroup) continue @@ -121,8 +119,6 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( dbReceipts := tdp.txsGrouper.groupReceipts(header, pool.Receipts) dbSCResults := tdp.scrsProc.processSCRs(body, header, pool.Scrs, numOfShards) - tdp.scrsProc.addScrsReceiverToAlteredAccounts(alteredAccounts, dbSCResults, header.GetShardID(), numOfShards) - srcsNoTxInCurrentShard := tdp.scrsDataToTxs.attachSCRsToTransactionsAndReturnSCRsWithoutTx(normalTxs, dbSCResults) tdp.scrsDataToTxs.processTransactionsAfterSCRsWereAttached(normalTxs) txHashStatus, txHashFee := tdp.scrsDataToTxs.processSCRsWithoutTx(srcsNoTxInCurrentShard) @@ -135,7 +131,6 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( Transactions: txsSlice, ScResults: dbSCResults, Receipts: dbReceipts, - AlteredAccts: alteredAccounts, TxHashStatus: txHashStatus, TxHashFee: txHashFee, } diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 69bfb37c..e32f0888 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -1,8 +1,6 @@ package transactions import ( - "encoding/hex" - "fmt" "math/big" "testing" @@ -30,39 +28,6 @@ func createMockArgsTxsDBProc() *ArgsTransactionProcessor { return args } -func TestAddToAlteredAddresses(t *testing.T) { - t.Parallel() - - sender := "senderAddress" - receiver := "receiverAddress" - tx := &data.Transaction{ - Sender: sender, - Receiver: receiver, - Data: []byte("ESDTTransfer@31323334352d373066366534@174876e800"), - } - alteredAddress := data.NewAlteredAccounts() - selfShardID := uint32(0) - mb := &block.MiniBlock{} - - grouper := txsGrouper{ - txBuilder: &dbTransactionBuilder{}, - } - grouper.addToAlteredAddresses(tx, alteredAddress, mb, selfShardID, false) - - alteredAccounts, ok := alteredAddress.Get(receiver) - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - BalanceChange: true, - }, alteredAccounts[0]) - - alteredAccounts, ok = alteredAddress.Get(sender) - require.True(t, ok) - require.Equal(t, &data.AlteredAccount{ - BalanceChange: true, - IsSender: true, - }, alteredAccounts[0]) -} - func TestIsSCRForSenderWithGasUsed(t *testing.T) { t.Parallel() @@ -289,129 +254,6 @@ func TestSetTransactionSearchOrder(t *testing.T) { assert.True(t, txPoolHasSearchOrder(transactions, 1)) } -func TestAlteredAddresses(t *testing.T) { - expectedAlteredAccounts := make(map[string]struct{}) - // addresses marked with a comment should be added to the altered addresses map - - // normal txs - address1 := []byte("address1") // should be added - address2 := []byte("address2") - expectedAlteredAccounts[hex.EncodeToString(address1)] = struct{}{} - tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: address1, - RcvAddr: address2, - }, 0, big.NewInt(0)) - tx1Hash := []byte("tx1Hash") - - address3 := []byte("address3") - address4 := []byte("address4") // should be added - expectedAlteredAccounts[hex.EncodeToString(address4)] = struct{}{} - tx2 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: address3, - RcvAddr: address4, - }, 0, big.NewInt(0)) - tx2Hash := []byte("tx2hash") - - txMiniBlock1 := &block.MiniBlock{ - Type: block.TxBlock, - TxHashes: [][]byte{tx1Hash}, - SenderShardID: 0, - ReceiverShardID: 1, - } - txMiniBlock2 := &block.MiniBlock{ - Type: block.TxBlock, - TxHashes: [][]byte{tx2Hash}, - SenderShardID: 1, - ReceiverShardID: 0, - } - - // reward txs - address5 := []byte("address5") // should be added - expectedAlteredAccounts[hex.EncodeToString(address5)] = struct{}{} - rwdTx1 := &rewardTx.RewardTx{ - RcvAddr: address5, - } - rwdTx1Hash := []byte("rwdTx1") - - address6 := []byte("address6") - rwdTx2 := &rewardTx.RewardTx{ - RcvAddr: address6, - } - rwdTx2Hash := []byte("rwdTx2") - - rewTxMiniBlock1 := &block.MiniBlock{ - Type: block.RewardsBlock, - TxHashes: [][]byte{rwdTx1Hash}, - SenderShardID: core.MetachainShardId, - ReceiverShardID: 0, - } - rewTxMiniBlock2 := &block.MiniBlock{ - Type: block.RewardsBlock, - TxHashes: [][]byte{rwdTx2Hash}, - SenderShardID: core.MetachainShardId, - ReceiverShardID: 1, - } - - // smart contract results - address7 := []byte("address7") // should be added - address8 := []byte("address8") - expectedAlteredAccounts[hex.EncodeToString(address7)] = struct{}{} - scr1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ - RcvAddr: address7, - SndAddr: address8, - }, 0, big.NewInt(0)) - scr1Hash := []byte("scr1Hash") - - address9 := []byte("address9") // should be added - address10 := []byte("address10") - expectedAlteredAccounts[hex.EncodeToString(address9)] = struct{}{} - scr2 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ - RcvAddr: address9, - SndAddr: address10, - }, 0, big.NewInt(0)) - scr2Hash := []byte("scr2Hash") - - scrMiniBlock1 := &block.MiniBlock{ - Type: block.SmartContractResultBlock, - TxHashes: [][]byte{scr1Hash, scr2Hash}, - SenderShardID: 1, - ReceiverShardID: 0, - } - - body := &block.Body{ - MiniBlocks: []*block.MiniBlock{txMiniBlock1, txMiniBlock2, rewTxMiniBlock1, rewTxMiniBlock2, scrMiniBlock1}, - } - - hdr := &block.Header{} - - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(tx1Hash): tx1, - string(tx2Hash): tx2, - }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scr1Hash): scr1, - string(scr2Hash): scr2, - }, - Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(rwdTx1Hash): outport.NewTransactionHandlerWithGasAndFee(rwdTx1, 0, big.NewInt(0)), - string(rwdTx2Hash): outport.NewTransactionHandlerWithGasAndFee(rwdTx2, 0, big.NewInt(0)), - }, - } - - args := createMockArgsTxsDBProc() - txDbProc, _ := NewTransactionsProcessor(args) - - results := txDbProc.PrepareTransactionsForDatabase(body, hdr, pool, false, 3) - - for addrActual := range results.AlteredAccts.GetAll() { - _, found := expectedAlteredAccounts[addrActual] - if !found { - assert.Fail(t, fmt.Sprintf("address %s not found", addrActual)) - } - } -} - func txPoolHasSearchOrder(txPool map[string]*data.Transaction, searchOrder uint32) bool { for _, tx := range txPool { if tx.SearchOrder == searchOrder { @@ -734,9 +576,6 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 2, len(res.ScResults)) - _, ok := res.AlteredAccts.Get("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u") - require.True(t, ok) - // transaction fail pool = &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ From 2108634cce1bdf889142631e2113a47b0bfdeaea Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 25 Oct 2022 11:35:24 +0300 Subject: [PATCH 71/96] fix integration tests --- integrationtests/accountsBalanceNftTransfer_test.go | 7 +++++++ .../accountsBalanceWithLowerTimestamp_test.go | 4 ++++ integrationtests/collectionsIndex_test.go | 3 ++- integrationtests/createNFTWithTags_test.go | 9 +++++---- .../testdata/claimRewards/tx-claim-rewards.json | 1 + 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 41d010ca..a14693e3 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -112,6 +112,13 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { coreAlteredAccounts = map[string]*outport.AlteredAccount{ addrHex: { Address: addrHex, + Tokens: []*outport.AccountTokenData{ + { + Identifier: "NFT-abcdef", + Nonce: 7440483, + Balance: "0", + }, + }, }, addrReceiverHex: { Address: addrReceiverHex, diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 66a4ad4f..a671efe4 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -125,6 +125,8 @@ func TestIndexAccountsBalance(t *testing.T) { } coreAlteredAccounts[encodedAddr].Balance = "2000" + coreAlteredAccounts[encodedAddr].BalanceChange = true + coreAlteredAccounts[encodedAddr].IsSender = true pool = &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ "h1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ @@ -187,6 +189,8 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Balance = "2000" coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" + coreAlteredAccounts[encodedAddr].BalanceChange = false + coreAlteredAccounts[encodedAddr].IsSender = false pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index e0297eae..3a70df83 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -176,7 +176,8 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { Address: addrHex, Tokens: []*outport.AccountTokenData{ { - Identifier: "NFT-abcdef", + Identifier: "SSSS-dddd", + Nonce: 22, Balance: "0", Properties: "ok", MetaData: &esdt.MetaData{ diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index cdb2329c..c8a0414e 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -72,10 +72,11 @@ func TestCreateNFTWithTags(t *testing.T) { Balance: "0", Tokens: []*outport.AccountTokenData{ { - Identifier: "DESK-abcd", - Nonce: 1, - Balance: "1000", - Properties: "ok", + IsNFTCreate: true, + Identifier: "DESK-abcd", + Nonce: 1, + Balance: "1000", + Properties: "ok", MetaData: &esdt.MetaData{ Creator: []byte("creator"), Attributes: []byte("tags:hello,something,do,music,art,gallery;metadata:QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud"), diff --git a/integrationtests/testdata/claimRewards/tx-claim-rewards.json b/integrationtests/testdata/claimRewards/tx-claim-rewards.json index 0f729fd3..3765344d 100644 --- a/integrationtests/testdata/claimRewards/tx-claim-rewards.json +++ b/integrationtests/testdata/claimRewards/tx-claim-rewards.json @@ -11,6 +11,7 @@ "gasLimit": 6000000, "gasUsed": 1068000, "fee": "78000000000000", + "hasOperations":true, "initialPaidFee":"127320000000000", "data": "Y2xhaW1SZXdhcmRz", "signature": "", From 88ffeebb57e5d0dc19c0386b646239c494b76945 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 26 Oct 2022 13:35:41 +0300 Subject: [PATCH 72/96] refactor python scrips --- scripts/observers/.env | 2 +- scripts/observers/README.md | 5 + scripts/observers/clean.py | 19 ++-- scripts/observers/config.py | 169 +++++++++++++++-------------- scripts/observers/requirements.txt | 1 + scripts/observers/start.py | 47 ++++---- scripts/observers/stop.py | 17 ++- 7 files changed, 149 insertions(+), 111 deletions(-) create mode 100644 scripts/observers/README.md create mode 100644 scripts/observers/requirements.txt diff --git a/scripts/observers/.env b/scripts/observers/.env index 4d17bcfd..74ac0e34 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -4,5 +4,5 @@ ELROND_CONFIG_BRANCH="extend-config-toml-for-t.1.40" ELROND_GO_URL="https://github.com/ElrondNetwork/elrond-go" ELROND_GO_BRANCH="feat/altered-accounts" -WORKING_DIRECTORY="/IndexerObservers/" +WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR="observer_shard_" diff --git a/scripts/observers/README.md b/scripts/observers/README.md new file mode 100644 index 00000000..9e53eb58 --- /dev/null +++ b/scripts/observers/README.md @@ -0,0 +1,5 @@ +## Prerequisites + +``` +pip3 install -r ./requirements.txt +``` \ No newline at end of file diff --git a/scripts/observers/clean.py b/scripts/observers/clean.py index 9dcd567a..99bbf716 100644 --- a/scripts/observers/clean.py +++ b/scripts/observers/clean.py @@ -3,12 +3,17 @@ from pathlib import Path from dotenv import load_dotenv -load_dotenv() -working_dir = str(Path.home()) + str(os.getenv('WORKING_DIRECTORY')) +def main(): + load_dotenv() + working_dir = Path(Path.home() / os.getenv('WORKING_DIRECTORY')) -try: - shutil.rmtree(working_dir) - print("done") -except: - print('did nothing') + try: + shutil.rmtree(working_dir) + print(f"removed directory: {working_dir}") + except FileNotFoundError: + print("nothing to clean") + + +if __name__ == "__main__": + main() diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 3f738623..6692117f 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -1,4 +1,7 @@ import os +import stat +import subprocess + import toml import sys import shutil @@ -6,108 +9,116 @@ from git import Repo from dotenv import load_dotenv -load_dotenv() - -observer_dir = str(os.getenv('OBSERVER_DIR')) METACHAIN = 4294967295 +WS_PORT_BASE = 22111 def update_toml_indexer(path, shard_id): # prefs.toml - data1 = toml.load(path+"/prefs.toml") - data1['config']['web-socket']['server-url'] = str(shard_id) + prefs_data = toml.load(path + "/prefs.toml") + prefs_data['config']['web-socket']['server-url'] = str(shard_id) if shard_id != METACHAIN: - data1['config']['web-socket']['server-url'] = "localhost:" + str(22111+shard_id) + prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_PORT_BASE + shard_id) else: - data1['config']['web-socket']['server-url'] = "localhost:" + str(22111+10) - f = open(path+"/prefs.toml", 'w') - toml.dump(data1, f) + prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_PORT_BASE + 10) + f = open(path + "/prefs.toml", 'w') + toml.dump(prefs_data, f) f.close() def update_toml_node(path, shard_id): # prefs.toml - data1 = toml.load(path+"/prefs.toml") - data1['Preferences']['DestinationShardAsObserver'] = str(shard_id) - f = open(path+"/prefs.toml", 'w') - toml.dump(data1, f) + prefs_data = toml.load(path + "/prefs.toml") + prefs_data['Preferences']['DestinationShardAsObserver'] = str(shard_id) + f = open(path + "/prefs.toml", 'w') + toml.dump(prefs_data, f) f.close() # external.toml - data2 = toml.load(path+"/external.toml") - data2['WebSocketConnector']['Enabled'] = True + external_data = toml.load(path + "/external.toml") + external_data['WebSocketConnector']['Enabled'] = True if shard_id != METACHAIN: - data2['WebSocketConnector']['URL'] = "localhost:" + str(22111+shard_id) + external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_PORT_BASE + shard_id) else: - data2['WebSocketConnector']['URL'] = "localhost:" + str(22111+10) - f = open(path+"/external.toml", 'w') - toml.dump(data2, f) + external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_PORT_BASE + 10) + f = open(path + "/external.toml", 'w') + toml.dump(external_data, f) f.close() -def prepare_observer(shard_id): +def prepare_observer(shard_id, working_dir, config_folder): + observer_dir = str(os.getenv('OBSERVER_DIR')) current_observer = observer_dir + str(shard_id) - working_dir_observer = working_dir + current_observer + working_dir_observer = Path(working_dir / current_observer) os.mkdir(working_dir_observer) - os.mkdir(working_dir_observer + "/indexer") - os.mkdir(working_dir_observer + "/node") + os.mkdir(Path(working_dir_observer / "indexer")) + os.mkdir(Path(working_dir_observer / "node")) - node_config = working_dir_observer + "/node/config" - indexer_config = working_dir_observer + "/indexer/config" + node_config = Path(working_dir_observer / "node" / "config") + indexer_config = Path(working_dir_observer / "indexer" / "config") shutil.copytree(config_folder, node_config) shutil.copytree("../../cmd/elasticindexer/config", indexer_config) - shutil.copyfile("../../cmd/elasticindexer/elasticindexer", working_dir_observer+"/indexer/elasticindexer") - os.system("chmod +x " + working_dir_observer+"/indexer/elasticindexer") - shutil.copyfile(working_dir + "/elrond-go/cmd/node/node", working_dir_observer+"/node/node") - os.system("chmod +x " + working_dir_observer+"/node/node") - update_toml_node(node_config, shard_id) - update_toml_indexer(indexer_config, shard_id) - - -working_dir = str(Path.home()) + str(os.getenv('WORKING_DIRECTORY')) -try: - os.makedirs(working_dir) -except FileExistsError: - print("working directory already exists") - print("use `python3 clean.py` command first") - sys.exit() - -# CLONE elrond-config -print("cloning elrond-config....") -config_folder = working_dir + "/config" -if not os.path.isdir(config_folder): - Repo.clone_from(os.getenv('ELROND_CONFIG_URL'), config_folder) - -repo_cfg = Repo(config_folder) -repo_cfg.git.checkout(os.getenv('ELROND_CONFIG_BRANCH')) - -# CLONE elrond-go -print("cloning elrond-go....") -elrond_go_folder = working_dir + "/elrond-go" -if not os.path.isdir(elrond_go_folder): - Repo.clone_from(os.getenv('ELROND_GO_URL'), elrond_go_folder) - -repo_elrond_go = Repo(elrond_go_folder) -repo_elrond_go.git.checkout(os.getenv('ELROND_GO_BRANCH')) - -# build binary elrond-go -print("building node...") -current_dir = os.getcwd() -os.chdir(elrond_go_folder + "/cmd/node") -os.system("go build") -os.chdir(current_dir) - -# build binary indexer -print("building indexer...") -os.chdir("../../cmd/elasticindexer") -os.system("go build") - -os.chdir(current_dir) - -# prepare observers -print("preparing config...") -prepare_observer(0) -prepare_observer(1) -prepare_observer(2) -prepare_observer(4294967295) + shutil.copyfile("../../cmd/elasticindexer/elasticindexer", Path(working_dir_observer / "indexer/elasticindexer")) + + elastic_indexer_exec = Path(working_dir_observer / "indexer/elasticindexer") + st = os.stat(elastic_indexer_exec) + os.chmod(elastic_indexer_exec, st.st_mode | stat.S_IEXEC) + + shutil.copyfile(Path(working_dir / "elrond-go/cmd/node/node"), Path(working_dir_observer / "node/node")) + + node_exec_path = Path(working_dir_observer / "node/node") + st = os.stat(node_exec_path) + os.chmod(node_exec_path, st.st_mode | stat.S_IEXEC) + + update_toml_node(str(node_config), shard_id) + update_toml_indexer(str(indexer_config), shard_id) + + +def main(): + load_dotenv() + working_dir = Path(Path.home() / os.getenv('WORKING_DIRECTORY')) + + try: + os.makedirs(working_dir) + except FileExistsError: + print(f"working directory {working_dir} already exists") + print("use `python3 clean.py` command first") + sys.exit() + + # CLONE elrond-config + print("cloning elrond-config....") + config_folder = Path(working_dir / "config") + if not os.path.isdir(config_folder): + Repo.clone_from(os.getenv('ELROND_CONFIG_URL'), config_folder) + + repo_cfg = Repo(config_folder) + repo_cfg.git.checkout(os.getenv('ELROND_CONFIG_BRANCH')) + + # CLONE elrond-go + print("cloning elrond-go....") + elrond_go_folder = Path(working_dir / "elrond-go") + if not os.path.isdir(elrond_go_folder): + Repo.clone_from(os.getenv('ELROND_GO_URL'), elrond_go_folder) + + repo_elrond_go = Repo(elrond_go_folder) + repo_elrond_go.git.checkout(os.getenv('ELROND_GO_BRANCH')) + + # build binary elrond-go + print("building node...") + subprocess.check_call(["go", "build"], cwd=Path(elrond_go_folder / "cmd/node")) + + # build binary indexer + print("building indexer...") + subprocess.check_call(["go", "build"], cwd="../../cmd/elasticindexer") + + # prepare observers + print("preparing config...") + prepare_observer(0, working_dir, config_folder) + prepare_observer(1, working_dir, config_folder) + prepare_observer(2, working_dir, config_folder) + prepare_observer(METACHAIN, working_dir, config_folder) + + +if __name__ == "__main__": + main() diff --git a/scripts/observers/requirements.txt b/scripts/observers/requirements.txt new file mode 100644 index 00000000..0899c299 --- /dev/null +++ b/scripts/observers/requirements.txt @@ -0,0 +1 @@ +git \ No newline at end of file diff --git a/scripts/observers/start.py b/scripts/observers/start.py index e9464532..c638bde6 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -3,39 +3,42 @@ from pathlib import Path from dotenv import load_dotenv -load_dotenv() -working_dir = str(Path.home()) + str(os.getenv('WORKING_DIRECTORY')) -observer_dir = str(os.getenv('OBSERVER_DIR')) - - -def start_observer(shard_id): - current_observer = observer_dir + str(shard_id) - working_dir_observer = working_dir + current_observer +def start_observer(shard_id, working_dir): + current_observer = str(os.getenv('OBSERVER_DIR')) + str(shard_id) + working_dir_observer = Path(working_dir / current_observer) current_directory = os.getcwd() # start observer - os.chdir(working_dir_observer + "/node") + os.chdir(Path(working_dir_observer / "node")) command = "./node" + " --log-level *:DEBUG --no-key --log-save" - # os.system("gnome-terminal 'bash -c \"" + command + ";bash\"'") os.system("screen -d -m -S obs" + str(shard_id) + " " + command) - os.chdir(current_directory) + # start indexer - os.chdir(working_dir_observer + "/indexer") + os.chdir(Path(working_dir_observer / "indexer")) command = "./elasticindexer" + " --log-level *:DEBUG --log-save" - # os.system("gnome-terminal 'bash -c \"" + command + ";bash\"'") os.system("screen -d -m -S indexer" + str(shard_id) + " " + command) + os.chdir(current_directory) +def main(): + load_dotenv() + + working_dir = Path(Path.home()) / os.getenv('WORKING_DIRECTORY') + if not os.path.exists(working_dir): + print("working directory folder is missing...you should run first `python3 config.py` command") + sys.exit() + + print("staring observers and indexers....") + + start_observer(0, working_dir) + start_observer(1, working_dir) + start_observer(2, working_dir) + start_observer(4294967295, working_dir) + + print("done") -if not os.path.exists(working_dir): - print("working directory folder is missing...you should run first `python3 config.py` command") - sys.exit() -print("staring observers and indexers....") +if __name__ == "__main__": + main() -start_observer(0) -start_observer(1) -start_observer(2) -start_observer(4294967295) -print("done") diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py index a62f905b..b2d6362f 100644 --- a/scripts/observers/stop.py +++ b/scripts/observers/stop.py @@ -1,4 +1,17 @@ import os -os.system("killall screen") -print("done") + +def main(): + os.system("screen -X -S obs0 quit") + os.system("screen -X -S obs1 quit") + os.system("screen -X -S obs2 quit") + os.system("screen -X -S obs4294967295 quit") + os.system("screen -X -S indexer0 quit") + os.system("screen -X -S indexer1 quit") + os.system("screen -X -S indexer2 quit") + os.system("screen -X -S indexer4294967295 quit") + print("done") + + +if __name__ == "__main__": + main() From 834646263968f9d8ec4442eceb934f78d2c0ffa6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 26 Oct 2022 13:37:01 +0300 Subject: [PATCH 73/96] small fix --- scripts/observers/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/observers/start.py b/scripts/observers/start.py index c638bde6..ed9796a0 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -21,6 +21,7 @@ def start_observer(shard_id, working_dir): os.chdir(current_directory) + def main(): load_dotenv() @@ -41,4 +42,3 @@ def main(): if __name__ == "__main__": main() - From b3997bdb9bc709b125af19e50d280ee0bc781645 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 26 Oct 2022 16:08:11 +0300 Subject: [PATCH 74/96] fix unmarshall --- process/wsindexer/unmarshal.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process/wsindexer/unmarshal.go b/process/wsindexer/unmarshal.go index 74be7b36..abb23afc 100644 --- a/process/wsindexer/unmarshal.go +++ b/process/wsindexer/unmarshal.go @@ -170,13 +170,13 @@ func (i *indexer) getHeader(marshaledData []byte) (data.HeaderHandler, error) { switch headerTypeStruct.HeaderType { case core.MetaHeader: hStruct := struct { - H1 *block.Header `json:"Header"` + H1 *block.MetaBlock `json:"Header"` }{} err = i.marshaller.Unmarshal(&hStruct, marshaledData) return hStruct.H1, err case core.ShardHeaderV1: hStruct := struct { - H1 *block.MetaBlock `json:"Header"` + H1 *block.Header `json:"Header"` }{} err = i.marshaller.Unmarshal(&hStruct, marshaledData) return hStruct.H1, err From 6574353715ba555ba73980f53d2cad9e2e50353f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 26 Oct 2022 16:37:30 +0300 Subject: [PATCH 75/96] fix log processor --- process/elasticproc/logsevents/logsAndEventsProcessor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 5eead50a..feb0d46e 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -158,7 +158,7 @@ func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byt scr, ok := lep.logsData.scrsMap[logHashHexEncoded] if ok { scr.HasOperations = true - return + continue } if res.processed { From 5b1c19f16c3b6a801a6af7727915e38b2792e9d1 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Oct 2022 10:06:41 +0300 Subject: [PATCH 76/96] refactor --- scripts/observers/config.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 6692117f..730163aa 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -1,7 +1,6 @@ import os import stat import subprocess - import toml import sys import shutil @@ -11,37 +10,41 @@ METACHAIN = 4294967295 WS_PORT_BASE = 22111 +WS_METACHAIN_PORT = WS_PORT_BASE + 50 def update_toml_indexer(path, shard_id): # prefs.toml - prefs_data = toml.load(path + "/prefs.toml") + path_prefs = Path(path / "prefs.toml") + prefs_data = toml.load(str(path_prefs)) prefs_data['config']['web-socket']['server-url'] = str(shard_id) if shard_id != METACHAIN: prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_PORT_BASE + shard_id) else: - prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_PORT_BASE + 10) - f = open(path + "/prefs.toml", 'w') + prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_METACHAIN_PORT) + f = open(path_prefs, 'w') toml.dump(prefs_data, f) f.close() def update_toml_node(path, shard_id): # prefs.toml - prefs_data = toml.load(path + "/prefs.toml") + path_prefs = Path(path / "prefs.toml") + prefs_data = toml.load(str(path_prefs)) prefs_data['Preferences']['DestinationShardAsObserver'] = str(shard_id) - f = open(path + "/prefs.toml", 'w') + f = open(path_prefs, 'w') toml.dump(prefs_data, f) f.close() # external.toml - external_data = toml.load(path + "/external.toml") + path_external = Path(path / "external.toml") + external_data = toml.load(str(path_external)) external_data['WebSocketConnector']['Enabled'] = True if shard_id != METACHAIN: external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_PORT_BASE + shard_id) else: - external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_PORT_BASE + 10) - f = open(path + "/external.toml", 'w') + external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_METACHAIN_PORT) + f = open(path_external, 'w') toml.dump(external_data, f) f.close() @@ -71,8 +74,8 @@ def prepare_observer(shard_id, working_dir, config_folder): st = os.stat(node_exec_path) os.chmod(node_exec_path, st.st_mode | stat.S_IEXEC) - update_toml_node(str(node_config), shard_id) - update_toml_indexer(str(indexer_config), shard_id) + update_toml_node(node_config, shard_id) + update_toml_indexer(indexer_config, shard_id) def main(): From 2cd634fdd593cd50ce1476817dd781a79328e04d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Oct 2022 14:57:59 +0300 Subject: [PATCH 77/96] fixes after review --- process/elasticproc/logsevents/interface.go | 2 +- scripts/observers/.env | 2 +- scripts/observers/clean.py | 8 +++++++- scripts/observers/config.py | 8 ++++++-- scripts/observers/start.py | 2 +- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/process/elasticproc/logsevents/interface.go b/process/elasticproc/logsevents/interface.go index 20144c94..42c9a0a0 100644 --- a/process/elasticproc/logsevents/interface.go +++ b/process/elasticproc/logsevents/interface.go @@ -23,8 +23,8 @@ type argsProcessEvent struct { type argOutputProcessEvent struct { tokenInfo *data.TokenInfo delegator *data.Delegator - processed bool updatePropNFT *data.NFTDataUpdate + processed bool } type eventsProcessor interface { diff --git a/scripts/observers/.env b/scripts/observers/.env index 74ac0e34..b63414e8 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -5,4 +5,4 @@ ELROND_GO_URL="https://github.com/ElrondNetwork/elrond-go" ELROND_GO_BRANCH="feat/altered-accounts" WORKING_DIRECTORY="IndexerObservers" -OBSERVER_DIR="observer_shard_" +OBSERVER_DIR_PREFIX="observer_shard_" diff --git a/scripts/observers/clean.py b/scripts/observers/clean.py index 99bbf716..16ceab7a 100644 --- a/scripts/observers/clean.py +++ b/scripts/observers/clean.py @@ -1,12 +1,18 @@ import shutil import os +import sys from pathlib import Path from dotenv import load_dotenv def main(): load_dotenv() - working_dir = Path(Path.home() / os.getenv('WORKING_DIRECTORY')) + working_dir_var = os.getenv('WORKING_DIRECTORY') + if working_dir_var == "": + print("working directory folder name cannot be empty") + sys.exit() + + working_dir = Path(Path.home() / working_dir_var) try: shutil.rmtree(working_dir) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 730163aa..281349f5 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -50,7 +50,7 @@ def update_toml_node(path, shard_id): def prepare_observer(shard_id, working_dir, config_folder): - observer_dir = str(os.getenv('OBSERVER_DIR')) + observer_dir = str(os.getenv('OBSERVER_DIR_PREFIX')) current_observer = observer_dir + str(shard_id) working_dir_observer = Path(working_dir / current_observer) os.mkdir(working_dir_observer) @@ -80,7 +80,11 @@ def prepare_observer(shard_id, working_dir, config_folder): def main(): load_dotenv() - working_dir = Path(Path.home() / os.getenv('WORKING_DIRECTORY')) + working_dir_var = os.getenv('WORKING_DIRECTORY') + if working_dir_var == "": + print("working directory folder name cannot be empty") + sys.exit() + working_dir = Path(Path.home() / working_dir_var) try: os.makedirs(working_dir) diff --git a/scripts/observers/start.py b/scripts/observers/start.py index ed9796a0..cfe0a744 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -5,7 +5,7 @@ def start_observer(shard_id, working_dir): - current_observer = str(os.getenv('OBSERVER_DIR')) + str(shard_id) + current_observer = str(os.getenv('OBSERVER_DIR_PREFIX')) + str(shard_id) working_dir_observer = Path(working_dir / current_observer) current_directory = os.getcwd() From c1d952034ea9d15415c8bfaa8e767bfaaaffc156 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Oct 2022 15:12:53 +0300 Subject: [PATCH 78/96] fixes --- scripts/observers/clean.py | 12 ++---------- scripts/observers/config.py | 8 ++------ scripts/observers/start.py | 4 ++-- scripts/observers/utils.py | 12 ++++++++++++ 4 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 scripts/observers/utils.py diff --git a/scripts/observers/clean.py b/scripts/observers/clean.py index 16ceab7a..0c6df63a 100644 --- a/scripts/observers/clean.py +++ b/scripts/observers/clean.py @@ -1,19 +1,11 @@ import shutil -import os -import sys -from pathlib import Path from dotenv import load_dotenv +from utils import get_working_dir def main(): load_dotenv() - working_dir_var = os.getenv('WORKING_DIRECTORY') - if working_dir_var == "": - print("working directory folder name cannot be empty") - sys.exit() - - working_dir = Path(Path.home() / working_dir_var) - + working_dir = get_working_dir() try: shutil.rmtree(working_dir) print(f"removed directory: {working_dir}") diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 281349f5..8af9b7a5 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -7,6 +7,7 @@ from pathlib import Path from git import Repo from dotenv import load_dotenv +from utils import get_working_dir METACHAIN = 4294967295 WS_PORT_BASE = 22111 @@ -80,12 +81,7 @@ def prepare_observer(shard_id, working_dir, config_folder): def main(): load_dotenv() - working_dir_var = os.getenv('WORKING_DIRECTORY') - if working_dir_var == "": - print("working directory folder name cannot be empty") - sys.exit() - working_dir = Path(Path.home() / working_dir_var) - + working_dir = get_working_dir() try: os.makedirs(working_dir) except FileExistsError: diff --git a/scripts/observers/start.py b/scripts/observers/start.py index cfe0a744..7f1b0e08 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -2,6 +2,7 @@ import sys from pathlib import Path from dotenv import load_dotenv +from utils import get_working_dir def start_observer(shard_id, working_dir): @@ -24,8 +25,7 @@ def start_observer(shard_id, working_dir): def main(): load_dotenv() - - working_dir = Path(Path.home()) / os.getenv('WORKING_DIRECTORY') + working_dir = get_working_dir() if not os.path.exists(working_dir): print("working directory folder is missing...you should run first `python3 config.py` command") sys.exit() diff --git a/scripts/observers/utils.py b/scripts/observers/utils.py new file mode 100644 index 00000000..68d8f721 --- /dev/null +++ b/scripts/observers/utils.py @@ -0,0 +1,12 @@ +import os +import sys +from pathlib import Path + + +def get_working_dir(): + working_dir_var = os.getenv('WORKING_DIRECTORY') + if working_dir_var == "": + print("working directory folder name cannot be empty") + sys.exit() + + return Path(Path.home() / working_dir_var) From 833d50a08501e7fbcdd9f1da01c08da79f6469ff Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Oct 2022 15:44:28 +0300 Subject: [PATCH 79/96] refactor --- scripts/observers/clean.py | 2 +- scripts/observers/config.py | 9 +-------- scripts/observers/start.py | 7 ++----- scripts/observers/utils.py | 5 +++++ 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/scripts/observers/clean.py b/scripts/observers/clean.py index 0c6df63a..bc78d5d8 100644 --- a/scripts/observers/clean.py +++ b/scripts/observers/clean.py @@ -1,6 +1,6 @@ import shutil from dotenv import load_dotenv -from utils import get_working_dir +from utils import * def main(): diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 8af9b7a5..18c190bc 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -1,17 +1,10 @@ -import os import stat import subprocess import toml -import sys import shutil -from pathlib import Path from git import Repo from dotenv import load_dotenv -from utils import get_working_dir - -METACHAIN = 4294967295 -WS_PORT_BASE = 22111 -WS_METACHAIN_PORT = WS_PORT_BASE + 50 +from utils import * def update_toml_indexer(path, shard_id): diff --git a/scripts/observers/start.py b/scripts/observers/start.py index 7f1b0e08..9932eeee 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -1,8 +1,5 @@ -import os -import sys -from pathlib import Path from dotenv import load_dotenv -from utils import get_working_dir +from utils import * def start_observer(shard_id, working_dir): @@ -35,7 +32,7 @@ def main(): start_observer(0, working_dir) start_observer(1, working_dir) start_observer(2, working_dir) - start_observer(4294967295, working_dir) + start_observer(METACHAIN, working_dir) print("done") diff --git a/scripts/observers/utils.py b/scripts/observers/utils.py index 68d8f721..42447d22 100644 --- a/scripts/observers/utils.py +++ b/scripts/observers/utils.py @@ -3,6 +3,11 @@ from pathlib import Path +METACHAIN = 4294967295 +WS_PORT_BASE = 22111 +WS_METACHAIN_PORT = WS_PORT_BASE + 50 + + def get_working_dir(): working_dir_var = os.getenv('WORKING_DIRECTORY') if working_dir_var == "": From 3fed07b8f208ee1b69bb83645ee29cc7a50cfd73 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Oct 2022 16:42:30 +0300 Subject: [PATCH 80/96] update elrond go core --- go.mod | 2 +- go.sum | 6 ++---- process/elasticproc/accounts/accountsProcessor.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 97de075f..1fcac5c4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024142346-a31256ced50b + github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221027134105-3c82a6063029 github.com/ElrondNetwork/elrond-go-logger v1.0.8 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 6c08bcd7..52438202 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024141157-ab57db14b806 h1:/IZ0Z3DK5xpfbAxGrcOpmF4UwQ8NhXBzUoa9YZBGrE8= -github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024141157-ab57db14b806/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024142346-a31256ced50b h1:xCVyWzdljqb/477iNtdAon7Ixh/2H6oB5ZetQGBNas4= -github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221024142346-a31256ced50b/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221027134105-3c82a6063029 h1:Xm92W0p0qnFsneErpujb1FTQG8cJWYJRlyXu1QnPEF4= +github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221027134105-3c82a6063029/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index 7808185a..5df8d4fd 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -64,7 +64,7 @@ func splitAlteredAccounts( accountsToIndexESDT := make([]*data.AccountESDT, 0) //if the balance of the ESDT receiver is 0 the receiver is a new account most probably, and we should index it - ignoreAddress := !account.BalanceChange && notZeroBalance(account.Balance) && !account.IsSender + ignoreAddress := !account.BalanceChanged && notZeroBalance(account.Balance) && !account.IsSender if !ignoreAddress { regularAccountsToIndex = append(regularAccountsToIndex, &data.Account{ UserAccount: account, From b57115b478512e834b9937c476f337cbfba30dae Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Oct 2022 16:57:11 +0300 Subject: [PATCH 81/96] remove Path cast --- scripts/observers/config.py | 28 ++++++++++++++-------------- scripts/observers/start.py | 6 +++--- scripts/observers/utils.py | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 18c190bc..80b6f516 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -9,7 +9,7 @@ def update_toml_indexer(path, shard_id): # prefs.toml - path_prefs = Path(path / "prefs.toml") + path_prefs = path / "prefs.toml" prefs_data = toml.load(str(path_prefs)) prefs_data['config']['web-socket']['server-url'] = str(shard_id) if shard_id != METACHAIN: @@ -23,7 +23,7 @@ def update_toml_indexer(path, shard_id): def update_toml_node(path, shard_id): # prefs.toml - path_prefs = Path(path / "prefs.toml") + path_prefs = path / "prefs.toml" prefs_data = toml.load(str(path_prefs)) prefs_data['Preferences']['DestinationShardAsObserver'] = str(shard_id) f = open(path_prefs, 'w') @@ -31,7 +31,7 @@ def update_toml_node(path, shard_id): f.close() # external.toml - path_external = Path(path / "external.toml") + path_external = path / "external.toml" external_data = toml.load(str(path_external)) external_data['WebSocketConnector']['Enabled'] = True if shard_id != METACHAIN: @@ -46,25 +46,25 @@ def update_toml_node(path, shard_id): def prepare_observer(shard_id, working_dir, config_folder): observer_dir = str(os.getenv('OBSERVER_DIR_PREFIX')) current_observer = observer_dir + str(shard_id) - working_dir_observer = Path(working_dir / current_observer) + working_dir_observer = working_dir / current_observer os.mkdir(working_dir_observer) - os.mkdir(Path(working_dir_observer / "indexer")) - os.mkdir(Path(working_dir_observer / "node")) + os.mkdir(working_dir_observer / "indexer") + os.mkdir(working_dir_observer / "node") - node_config = Path(working_dir_observer / "node" / "config") - indexer_config = Path(working_dir_observer / "indexer" / "config") + node_config = working_dir_observer / "node" / "config" + indexer_config = working_dir_observer / "indexer" / "config" shutil.copytree(config_folder, node_config) shutil.copytree("../../cmd/elasticindexer/config", indexer_config) - shutil.copyfile("../../cmd/elasticindexer/elasticindexer", Path(working_dir_observer / "indexer/elasticindexer")) + shutil.copyfile("../../cmd/elasticindexer/elasticindexer", working_dir_observer / "indexer/elasticindexer") elastic_indexer_exec = Path(working_dir_observer / "indexer/elasticindexer") st = os.stat(elastic_indexer_exec) os.chmod(elastic_indexer_exec, st.st_mode | stat.S_IEXEC) - shutil.copyfile(Path(working_dir / "elrond-go/cmd/node/node"), Path(working_dir_observer / "node/node")) + shutil.copyfile(working_dir / "elrond-go/cmd/node/node", working_dir_observer / "node/node") - node_exec_path = Path(working_dir_observer / "node/node") + node_exec_path = working_dir_observer / "node/node" st = os.stat(node_exec_path) os.chmod(node_exec_path, st.st_mode | stat.S_IEXEC) @@ -84,7 +84,7 @@ def main(): # CLONE elrond-config print("cloning elrond-config....") - config_folder = Path(working_dir / "config") + config_folder = working_dir / "config" if not os.path.isdir(config_folder): Repo.clone_from(os.getenv('ELROND_CONFIG_URL'), config_folder) @@ -93,7 +93,7 @@ def main(): # CLONE elrond-go print("cloning elrond-go....") - elrond_go_folder = Path(working_dir / "elrond-go") + elrond_go_folder = working_dir / "elrond-go" if not os.path.isdir(elrond_go_folder): Repo.clone_from(os.getenv('ELROND_GO_URL'), elrond_go_folder) @@ -102,7 +102,7 @@ def main(): # build binary elrond-go print("building node...") - subprocess.check_call(["go", "build"], cwd=Path(elrond_go_folder / "cmd/node")) + subprocess.check_call(["go", "build"], cwd=elrond_go_folder / "cmd/node") # build binary indexer print("building indexer...") diff --git a/scripts/observers/start.py b/scripts/observers/start.py index 9932eeee..e57ff4de 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -4,16 +4,16 @@ def start_observer(shard_id, working_dir): current_observer = str(os.getenv('OBSERVER_DIR_PREFIX')) + str(shard_id) - working_dir_observer = Path(working_dir / current_observer) + working_dir_observer = working_dir / current_observer current_directory = os.getcwd() # start observer - os.chdir(Path(working_dir_observer / "node")) + os.chdir(working_dir_observer / "node") command = "./node" + " --log-level *:DEBUG --no-key --log-save" os.system("screen -d -m -S obs" + str(shard_id) + " " + command) # start indexer - os.chdir(Path(working_dir_observer / "indexer")) + os.chdir(working_dir_observer / "indexer") command = "./elasticindexer" + " --log-level *:DEBUG --log-save" os.system("screen -d -m -S indexer" + str(shard_id) + " " + command) diff --git a/scripts/observers/utils.py b/scripts/observers/utils.py index 42447d22..930a61dc 100644 --- a/scripts/observers/utils.py +++ b/scripts/observers/utils.py @@ -14,4 +14,4 @@ def get_working_dir(): print("working directory folder name cannot be empty") sys.exit() - return Path(Path.home() / working_dir_var) + return Path.home() / working_dir_var From 0359f0c27a5d0cde717073abf6a5c55aa5d9f27d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Oct 2022 16:59:07 +0300 Subject: [PATCH 82/96] fix integration tests --- integrationtests/accountsBalanceWithLowerTimestamp_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index a671efe4..314a7276 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -125,7 +125,7 @@ func TestIndexAccountsBalance(t *testing.T) { } coreAlteredAccounts[encodedAddr].Balance = "2000" - coreAlteredAccounts[encodedAddr].BalanceChange = true + coreAlteredAccounts[encodedAddr].BalanceChanged = true coreAlteredAccounts[encodedAddr].IsSender = true pool = &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ @@ -189,7 +189,7 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Balance = "2000" coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" - coreAlteredAccounts[encodedAddr].BalanceChange = false + coreAlteredAccounts[encodedAddr].BalanceChanged = false coreAlteredAccounts[encodedAddr].IsSender = false pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) From ed60ee5200be74f73cd66a0fa7650bcf556487ac Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 28 Oct 2022 15:09:57 +0300 Subject: [PATCH 83/96] latest elrond-go-core --- go.mod | 2 +- go.sum | 4 ++-- .../elasticproc/accounts/accountsProcessor.go | 22 ++++++++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 1fcac5c4..3e56c953 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221027134105-3c82a6063029 + github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221028092924-4f1c38bae5f9 github.com/ElrondNetwork/elrond-go-logger v1.0.8 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 52438202..8c3e3794 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221027134105-3c82a6063029 h1:Xm92W0p0qnFsneErpujb1FTQG8cJWYJRlyXu1QnPEF4= -github.com/ElrondNetwork/elrond-go-core v1.1.22-0.20221027134105-3c82a6063029/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221028092924-4f1c38bae5f9 h1:CO0KsXlVAwSUAU150TlH/c+xww4Ai30KtSU2cTo1OuY= +github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221028092924-4f1c38bae5f9/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index 5df8d4fd..2dddeee6 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -63,23 +63,33 @@ func splitAlteredAccounts( regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) + isSender, balanceChanged := false, false + if account.AdditionalData != nil { + isSender, balanceChanged = account.AdditionalData.IsSender, account.AdditionalData.BalanceChanged + } + //if the balance of the ESDT receiver is 0 the receiver is a new account most probably, and we should index it - ignoreAddress := !account.BalanceChanged && notZeroBalance(account.Balance) && !account.IsSender + ignoreAddress := !balanceChanged && notZeroBalance(account.Balance) && !isSender if !ignoreAddress { regularAccountsToIndex = append(regularAccountsToIndex, &data.Account{ UserAccount: account, - IsSender: account.IsSender, + IsSender: isSender, }) } for _, info := range account.Tokens { - accountsToIndexESDT = append(accountsToIndexESDT, &data.AccountESDT{ + accountESDT := &data.AccountESDT{ Account: account, TokenIdentifier: info.Identifier, - IsSender: account.IsSender, NFTNonce: info.Nonce, - IsNFTCreate: info.IsNFTCreate, - }) + } + if info.AdditionalData != nil { + accountESDT.IsSender = account.AdditionalData.IsSender + accountESDT.IsNFTCreate = info.AdditionalData.IsNFTCreate + } + + accountsToIndexESDT = append(accountsToIndexESDT, accountESDT) + } return regularAccountsToIndex, accountsToIndexESDT From 85d86bb323aab054297fbf193a70bf370ff4620c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 28 Oct 2022 15:12:55 +0300 Subject: [PATCH 84/96] fix integration tests --- .../accountsBalanceWithLowerTimestamp_test.go | 12 ++++++++---- integrationtests/createNFTWithTags_test.go | 12 +++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 314a7276..8f50d16d 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -125,8 +125,10 @@ func TestIndexAccountsBalance(t *testing.T) { } coreAlteredAccounts[encodedAddr].Balance = "2000" - coreAlteredAccounts[encodedAddr].BalanceChanged = true - coreAlteredAccounts[encodedAddr].IsSender = true + coreAlteredAccounts[encodedAddr].AdditionalData = &outport.AdditionalAccountData{ + IsSender: true, + BalanceChanged: true, + } pool = &outport.Pool{ Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ "h1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ @@ -189,8 +191,10 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Balance = "2000" coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" - coreAlteredAccounts[encodedAddr].BalanceChanged = false - coreAlteredAccounts[encodedAddr].IsSender = false + coreAlteredAccounts[encodedAddr].AdditionalData = &outport.AdditionalAccountData{ + IsSender: true, + BalanceChanged: true, + } pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index c8a0414e..8bd90c87 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -72,11 +72,13 @@ func TestCreateNFTWithTags(t *testing.T) { Balance: "0", Tokens: []*outport.AccountTokenData{ { - IsNFTCreate: true, - Identifier: "DESK-abcd", - Nonce: 1, - Balance: "1000", - Properties: "ok", + AdditionalData: &outport.AdditionalAccountTokenData{ + IsNFTCreate: true, + }, + Identifier: "DESK-abcd", + Nonce: 1, + Balance: "1000", + Properties: "ok", MetaData: &esdt.MetaData{ Creator: []byte("creator"), Attributes: []byte("tags:hello,something,do,music,art,gallery;metadata:QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud"), From 60da603124c929036d9ce182792bab0f280d8a6c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 28 Oct 2022 15:20:17 +0300 Subject: [PATCH 85/96] fix --- integrationtests/accountsBalanceWithLowerTimestamp_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 8f50d16d..a4099aac 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -192,8 +192,8 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[encodedAddr].Balance = "2000" coreAlteredAccounts[encodedAddr].Tokens[0].Balance = "0" coreAlteredAccounts[encodedAddr].AdditionalData = &outport.AdditionalAccountData{ - IsSender: true, - BalanceChanged: true, + IsSender: false, + BalanceChanged: false, } pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) From 6f62eb469e795ea5597eee582eb7163cb5942e97 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 31 Oct 2022 10:08:48 +0200 Subject: [PATCH 86/96] fixes --- process/elasticproc/accounts/accountsProcessor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index 2dddeee6..d933abd4 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -82,9 +82,9 @@ func splitAlteredAccounts( Account: account, TokenIdentifier: info.Identifier, NFTNonce: info.Nonce, + IsSender: isSender, } if info.AdditionalData != nil { - accountESDT.IsSender = account.AdditionalData.IsSender accountESDT.IsNFTCreate = info.AdditionalData.IsNFTCreate } From d82347b19e9967a8932db77ab232f2914aa45abb Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 31 Oct 2022 11:22:07 +0200 Subject: [PATCH 87/96] extra log --- process/elasticproc/accounts/accountsProcessor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index d933abd4..f09ba968 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -66,6 +66,8 @@ func splitAlteredAccounts( isSender, balanceChanged := false, false if account.AdditionalData != nil { isSender, balanceChanged = account.AdditionalData.IsSender, account.AdditionalData.BalanceChanged + } else { + log.Debug("accountsProcessor.splitAlteredAccounts", "nil additional data") } //if the balance of the ESDT receiver is 0 the receiver is a new account most probably, and we should index it From 8d8cfb7e65bf10a1714b57597828fd7afcb4b75a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 31 Oct 2022 11:22:44 +0200 Subject: [PATCH 88/96] modify --- process/elasticproc/accounts/accountsProcessor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index f09ba968..1d3327ed 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -67,7 +67,7 @@ func splitAlteredAccounts( if account.AdditionalData != nil { isSender, balanceChanged = account.AdditionalData.IsSender, account.AdditionalData.BalanceChanged } else { - log.Debug("accountsProcessor.splitAlteredAccounts", "nil additional data") + log.Debug("accountsProcessor.splitAlteredAccounts - nil additional data") } //if the balance of the ESDT receiver is 0 the receiver is a new account most probably, and we should index it From 2aef3099b210f778a7d21f93753ed7395bca6a2a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 7 Nov 2022 13:48:39 +0200 Subject: [PATCH 89/96] remove unused stub --- mock/userAccountStub.go | 55 ----------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 mock/userAccountStub.go diff --git a/mock/userAccountStub.go b/mock/userAccountStub.go deleted file mode 100644 index 6e8c1d37..00000000 --- a/mock/userAccountStub.go +++ /dev/null @@ -1,55 +0,0 @@ -package mock - -import ( - "math/big" -) - -// UserAccountStub - -type UserAccountStub struct { - GetBalanceCalled func() *big.Int - GetNonceCalled func() uint64 - AddressBytesCalled func() []byte - RetrieveValueCalled func(key []byte) ([]byte, uint32, error) -} - -// IncreaseNonce - -func (u *UserAccountStub) IncreaseNonce(_ uint64) { -} - -// GetBalance - -func (u *UserAccountStub) GetBalance() *big.Int { - if u.GetBalanceCalled != nil { - return u.GetBalanceCalled() - } - return nil -} - -// RetrieveValue - -func (u *UserAccountStub) RetrieveValue(key []byte) ([]byte, uint32, error) { - if u.RetrieveValueCalled != nil { - return u.RetrieveValueCalled(key) - } - - return nil, 0, nil -} - -// AddressBytes - -func (u *UserAccountStub) AddressBytes() []byte { - if u.AddressBytesCalled != nil { - return u.AddressBytesCalled() - } - return nil -} - -// GetNonce - -func (u *UserAccountStub) GetNonce() uint64 { - if u.GetNonceCalled != nil { - return u.GetNonceCalled() - } - return 0 -} - -// IsInterfaceNil - -func (u *UserAccountStub) IsInterfaceNil() bool { - return false -} From 8e12c0500e12fd9fc98512c37a6e7bb89a7c0ff0 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Nov 2022 14:01:22 +0200 Subject: [PATCH 90/96] update core --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3e56c953..2bef6733 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221028092924-4f1c38bae5f9 + github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108113024-38dd797ed574 github.com/ElrondNetwork/elrond-go-logger v1.0.8 github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 8c3e3794..1af5bf60 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221028092924-4f1c38bae5f9 h1:CO0KsXlVAwSUAU150TlH/c+xww4Ai30KtSU2cTo1OuY= -github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221028092924-4f1c38bae5f9/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108113024-38dd797ed574 h1:XCj1Fo6Xl8EzAFhaIFcCY393Cu2HPppAKeOdxMaTFT4= +github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108113024-38dd797ed574/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= From 921346655fe0d690e715f3ffa08f607e954292aa Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Nov 2022 14:31:10 +0200 Subject: [PATCH 91/96] fixes --- .../elasticproc/accounts/accountsProcessor.go | 21 ++++------- .../accounts/accountsProcessor_test.go | 27 +++++++------- .../elasticproc/converters/tokenMetaData.go | 37 +++++++------------ .../converters/tokenMetaData_test.go | 26 ++++++------- .../elasticproc/logsevents/nftsProcessor.go | 19 +++++++++- 5 files changed, 63 insertions(+), 67 deletions(-) diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index 1d3327ed..33f6c20d 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -11,7 +11,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" - "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/outport" logger "github.com/ElrondNetwork/elrond-go-logger" ) @@ -235,25 +234,21 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In return big.NewInt(0), "", nil, nil } - esdtToken := &esdt.ESDigitalToken{} + accountTokenData := &outport.AccountTokenData{} for _, tokenData := range accountESDT.Account.Tokens { if tokenData.Identifier == accountESDT.TokenIdentifier && tokenData.Nonce == accountESDT.NFTNonce { - value, _ := big.NewInt(0).SetString(tokenData.Balance, 10) - esdtToken = &esdt.ESDigitalToken{ - Value: value, - Properties: []byte(tokenData.Properties), - TokenMetaData: tokenData.MetaData, - } + accountTokenData = tokenData } } - if esdtToken.Value == nil { + value, _ := big.NewInt(0).SetString(accountTokenData.Balance, 10) + if value == nil { return big.NewInt(0), "", nil, nil } - tokenMetaData := converters.PrepareTokenMetaData(ap.addressPubkeyConverter, esdtToken) + tokenMetaData := converters.PrepareTokenMetaData(accountTokenData.MetaData) - return esdtToken.Value, hex.EncodeToString(esdtToken.Properties), tokenMetaData, nil + return value, hex.EncodeToString([]byte(accountTokenData.Properties)), tokenMetaData, nil } // PutTokenMedataDataInTokens will put the TokenMedata in provided tokens data @@ -272,14 +267,14 @@ func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.Token continue } - tokenData.Data = converters.PrepareTokenMetaData(ap.addressPubkeyConverter, &esdt.ESDigitalToken{TokenMetaData: metadata}) + tokenData.Data = converters.PrepareTokenMetaData(metadata) } } func (ap *accountsProcessor) loadMetadataForToken( tokenData *data.TokenInfo, coreAlteredAccounts map[string]*outport.AlteredAccount, -) (*esdt.MetaData, error) { +) (*outport.TokenMetaData, error) { for _, account := range coreAlteredAccounts { for _, token := range account.Tokens { if tokenData.Token == token.Identifier && tokenData.Nonce == token.Nonce { diff --git a/process/elasticproc/accounts/accountsProcessor_test.go b/process/elasticproc/accounts/accountsProcessor_test.go index bab917c7..ddd620c9 100644 --- a/process/elasticproc/accounts/accountsProcessor_test.go +++ b/process/elasticproc/accounts/accountsProcessor_test.go @@ -14,7 +14,6 @@ import ( "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/converters" "github.com/ElrondNetwork/elastic-indexer-go/process/elasticproc/tags" "github.com/ElrondNetwork/elrond-go-core/core" - "github.com/ElrondNetwork/elrond-go-core/data/esdt" "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/stretchr/testify/require" ) @@ -146,7 +145,7 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { require.NotNil(t, ap) nftName := "Test-nft" - creator := []byte("010101") + creator := "010101" tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ @@ -160,9 +159,9 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { Balance: "1", Properties: "ok", Nonce: 10, - MetaData: &esdt.MetaData{ + MetaData: &outport.TokenMetaData{ Nonce: 10, - Name: []byte(nftName), + Name: nftName, Creator: creator, Royalties: 2, }, @@ -179,7 +178,7 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { require.Equal(t, hex.EncodeToString([]byte("ok")), prop) require.Equal(t, &data.TokenMetaData{ Name: nftName, - Creator: pubKeyConverter.Encode(creator), + Creator: creator, Royalties: 2, }, metaData) } @@ -305,8 +304,8 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { Identifier: "token", Nonce: 15, Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, }, { @@ -314,8 +313,8 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { Identifier: "token", Nonce: 16, Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, }, }, @@ -341,7 +340,7 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { Properties: hex.EncodeToString([]byte("ok")), TokenNonce: 15, Data: &data.TokenMetaData{ - Creator: "63726561746f72", + Creator: "creator", }, Timestamp: time.Duration(123), }, res[hex.EncodeToString([]byte(addr))+"-token-15"]) @@ -355,7 +354,7 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { Properties: hex.EncodeToString([]byte("ok")), TokenNonce: 16, Data: &data.TokenMetaData{ - Creator: "63726561746f72", + Creator: "creator", }, Timestamp: time.Duration(123), }, res[hex.EncodeToString([]byte(addr))+"-token-16"]) @@ -433,7 +432,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) - metadata0, metadata1 := &esdt.MetaData{Creator: []byte("creator 0")}, &esdt.MetaData{Creator: []byte("creator 1")} + metadata0, metadata1 := &outport.TokenMetaData{Creator: "creator 0"}, &outport.TokenMetaData{Creator: "creator 1"} tokensInfo := []*data.TokenInfo{ { Nonce: 5, @@ -465,7 +464,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { } ap.PutTokenMedataDataInTokens(tokensInfo, alteredAccounts) - require.Equal(t, hex.EncodeToString(metadata0.Creator), tokensInfo[0].Data.Creator) - require.Equal(t, hex.EncodeToString(metadata1.Creator), tokensInfo[1].Data.Creator) + require.Equal(t, metadata0.Creator, tokensInfo[0].Data.Creator) + require.Equal(t, metadata1.Creator, tokensInfo[1].Data.Creator) }) } diff --git a/process/elasticproc/converters/tokenMetaData.go b/process/elasticproc/converters/tokenMetaData.go index ae69b4c9..63f95a33 100644 --- a/process/elasticproc/converters/tokenMetaData.go +++ b/process/elasticproc/converters/tokenMetaData.go @@ -7,9 +7,7 @@ import ( "strings" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elrond-go-core/core" - "github.com/ElrondNetwork/elrond-go-core/core/check" - "github.com/ElrondNetwork/elrond-go-core/data/esdt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" ) const ( @@ -22,31 +20,22 @@ const ( ) // PrepareTokenMetaData will prepare the token metadata in a friendly format for database -func PrepareTokenMetaData(pubKeyConverter core.PubkeyConverter, esdtInfo *esdt.ESDigitalToken) *data.TokenMetaData { - if check.IfNil(pubKeyConverter) { +func PrepareTokenMetaData(tokenMetadata *outport.TokenMetaData) *data.TokenMetaData { + if tokenMetadata == nil { return nil } - if esdtInfo == nil || esdtInfo.TokenMetaData == nil { - return nil - } - - creatorStr := "" - if esdtInfo.TokenMetaData.Creator != nil { - creatorStr = pubKeyConverter.Encode(esdtInfo.TokenMetaData.Creator) - } - return &data.TokenMetaData{ - Name: string(esdtInfo.TokenMetaData.Name), - Creator: creatorStr, - Royalties: esdtInfo.TokenMetaData.Royalties, - Hash: esdtInfo.TokenMetaData.Hash, - URIs: esdtInfo.TokenMetaData.URIs, - Attributes: esdtInfo.TokenMetaData.Attributes, - Tags: ExtractTagsFromAttributes(esdtInfo.TokenMetaData.Attributes), - MetaData: ExtractMetaDataFromAttributes(esdtInfo.TokenMetaData.Attributes), - NonEmptyURIs: nonEmptyURIs(esdtInfo.TokenMetaData.URIs), - WhiteListedStorage: whiteListedStorage(esdtInfo.TokenMetaData.URIs), + Name: tokenMetadata.Name, + Creator: tokenMetadata.Creator, + Royalties: tokenMetadata.Royalties, + Hash: tokenMetadata.Hash, + URIs: tokenMetadata.URIs, + Attributes: tokenMetadata.Attributes, + Tags: ExtractTagsFromAttributes(tokenMetadata.Attributes), + MetaData: ExtractMetaDataFromAttributes(tokenMetadata.Attributes), + NonEmptyURIs: nonEmptyURIs(tokenMetadata.URIs), + WhiteListedStorage: whiteListedStorage(tokenMetadata.URIs), } } diff --git a/process/elasticproc/converters/tokenMetaData_test.go b/process/elasticproc/converters/tokenMetaData_test.go index fc33d76a..01cad44b 100644 --- a/process/elasticproc/converters/tokenMetaData_test.go +++ b/process/elasticproc/converters/tokenMetaData_test.go @@ -4,20 +4,18 @@ import ( "testing" "github.com/ElrondNetwork/elastic-indexer-go/data" - "github.com/ElrondNetwork/elastic-indexer-go/mock" - "github.com/ElrondNetwork/elrond-go-core/data/esdt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/stretchr/testify/require" ) func TestPrepareTokenMetaData(t *testing.T) { t.Parallel() - require.Nil(t, PrepareTokenMetaData(nil, nil)) - require.Nil(t, PrepareTokenMetaData(&mock.PubkeyConverterMock{}, nil)) + require.Nil(t, PrepareTokenMetaData(nil)) expectedTokenMetaData := &data.TokenMetaData{ Name: "token", - Creator: "63726561746f72", + Creator: "creator", Royalties: 0, Hash: []byte("hash"), URIs: [][]byte{[]byte("https://ipfs.io/ipfs/something"), []byte("uri")}, @@ -28,16 +26,14 @@ func TestPrepareTokenMetaData(t *testing.T) { WhiteListedStorage: true, } - result := PrepareTokenMetaData(&mock.PubkeyConverterMock{}, &esdt.ESDigitalToken{ - TokenMetaData: &esdt.MetaData{ - Nonce: 2, - Name: []byte("token"), - Creator: []byte("creator"), - Royalties: 0, - Hash: []byte("hash"), - URIs: [][]byte{[]byte(ipfsURL + "something"), []byte("uri")}, - Attributes: []byte("tags:test,free,fun;description:This is a test description for an awesome nft;metadata:metadata-test"), - }, + result := PrepareTokenMetaData(&outport.TokenMetaData{ + Nonce: 2, + Name: "token", + Creator: "creator", + Royalties: 0, + Hash: []byte("hash"), + URIs: [][]byte{[]byte(ipfsURL + "something"), []byte("uri")}, + Attributes: []byte("tags:test,free,fun;description:This is a test description for an awesome nft;metadata:metadata-test"), }) require.Equal(t, expectedTokenMetaData, result) diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index daec7189..13bffc99 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -10,6 +10,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/sharding" coreData "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/esdt" + "github.com/ElrondNetwork/elrond-go-core/data/outport" "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" ) @@ -143,7 +144,7 @@ func (np *nftsProcessor) processNFTEventOnSender( return } - tokenMetaData := converters.PrepareTokenMetaData(np.pubKeyConverter, esdtToken) + tokenMetaData := converters.PrepareTokenMetaData(np.convertMetaData(esdtToken.TokenMetaData)) tokensCreateInfo.Add(&data.TokenInfo{ Token: token, Identifier: converters.ComputeTokenIdentifier(token, nonceBig.Uint64()), @@ -152,3 +153,19 @@ func (np *nftsProcessor) processNFTEventOnSender( Nonce: nonceBig.Uint64(), }) } + +func (np *nftsProcessor) convertMetaData(metaData *esdt.MetaData) *outport.TokenMetaData { + if metaData == nil { + return nil + } + + return &outport.TokenMetaData{ + Nonce: metaData.Nonce, + Name: string(metaData.Name), + Creator: np.pubKeyConverter.Encode(metaData.Creator), + Royalties: metaData.Royalties, + Hash: metaData.Hash, + URIs: metaData.URIs, + Attributes: metaData.Attributes, + } +} From 69b6483b9fca59bd62ba03d9ca924b6a640eee2a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Nov 2022 14:36:15 +0200 Subject: [PATCH 92/96] fix integration tests --- integrationtests/accountsESDTRollback_test.go | 4 ++-- integrationtests/accountsESDTWithTokenType_test.go | 8 ++++---- integrationtests/collectionsIndex_test.go | 12 ++++++------ integrationtests/createNFTWithTags_test.go | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 76e98e5c..cb55975c 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -40,8 +40,8 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { Identifier: "TOKEN-eeee", Nonce: 2, Balance: "1000", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, Properties: "ok", }, diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 3eb28b83..f6ae14ae 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -76,8 +76,8 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { Balance: "1000", Nonce: 2, Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, }, }, @@ -149,8 +149,8 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) Nonce: 2, Balance: "1000", Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, }, }, diff --git a/integrationtests/collectionsIndex_test.go b/integrationtests/collectionsIndex_test.go index 3a70df83..1b4a99fc 100644 --- a/integrationtests/collectionsIndex_test.go +++ b/integrationtests/collectionsIndex_test.go @@ -73,8 +73,8 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { Balance: "1000", Nonce: 2, Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, }, }, @@ -88,8 +88,8 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { Balance: "1000", Nonce: 2, Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, }, }, @@ -180,8 +180,8 @@ func TestCollectionsIndexInsertAndDelete(t *testing.T) { Nonce: 22, Balance: "0", Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", }, }, }, diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 8bd90c87..1bed210c 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -79,8 +79,8 @@ func TestCreateNFTWithTags(t *testing.T) { Nonce: 1, Balance: "1000", Properties: "ok", - MetaData: &esdt.MetaData{ - Creator: []byte("creator"), + MetaData: &outport.TokenMetaData{ + Creator: "creator", Attributes: []byte("tags:hello,something,do,music,art,gallery;metadata:QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud"), }, }, From fba1d467bcd9e7740edd9982f16be23ad1098f0b Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Nov 2022 14:41:22 +0200 Subject: [PATCH 93/96] fix integration --- .../testdata/accountsESDTRollback/account-after-create.json | 2 +- .../accountsESDTWithTokenType/account-esdt-with-type.json | 2 +- .../accountsESDTWithTokenType/account-esdt-without-type.json | 2 +- .../testdata/accountsESDTWithTokenType/account-esdt.json | 2 +- .../createNFTWithTags/accounts-esdt-address-balance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integrationtests/testdata/accountsESDTRollback/account-after-create.json b/integrationtests/testdata/accountsESDTRollback/account-after-create.json index 99c1574e..ddb92726 100644 --- a/integrationtests/testdata/accountsESDTRollback/account-after-create.json +++ b/integrationtests/testdata/accountsESDTRollback/account-after-create.json @@ -4,7 +4,7 @@ "balance": "1000", "balanceNum": 1.0E-15, "data": { - "creator": "63726561746f72", + "creator": "creator", "whiteListedStorage": false, "nonEmptyURIs": false }, diff --git a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json index a9934b03..11b6ac38 100644 --- a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json +++ b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-with-type.json @@ -7,7 +7,7 @@ "tokenNonce": 2, "properties": "6f6b", "data": { - "creator": "63726561746f72", + "creator": "creator", "nonEmptyURIs": false, "whiteListedStorage": false }, diff --git a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json index 6d936d25..53df61ef 100644 --- a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json +++ b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt-without-type.json @@ -7,7 +7,7 @@ "tokenNonce": 2, "properties": "6f6b", "data": { - "creator": "63726561746f72", + "creator": "creator", "nonEmptyURIs": false, "whiteListedStorage": false }, diff --git a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json index abf69a78..b12737da 100644 --- a/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json +++ b/integrationtests/testdata/accountsESDTWithTokenType/account-esdt.json @@ -7,7 +7,7 @@ "tokenNonce": 2, "properties": "6f6b", "data": { - "creator": "63726561746f72", + "creator": "creator", "nonEmptyURIs": false, "whiteListedStorage": false }, diff --git a/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json b/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json index d113f36d..4a388de6 100644 --- a/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json +++ b/integrationtests/testdata/createNFTWithTags/accounts-esdt-address-balance.json @@ -4,7 +4,7 @@ "balance": "1000", "balanceNum": 1.0E-15, "data": { - "creator": "63726561746f72", + "creator": "creator", "metadata": "QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud", "whiteListedStorage": false, "attributes": "dGFnczpoZWxsbyxzb21ldGhpbmcsZG8sbXVzaWMsYXJ0LGdhbGxlcnk7bWV0YWRhdGE6UW1aMlFxYUdxNGJxc0V6czVKTFRqUm1tdlIyR0FSNHFYSlpCTjhpYmZEZGF1ZA==", From 4f66684121088e1006fe44590b375417746dc2c0 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 9 Nov 2022 11:32:35 +0200 Subject: [PATCH 94/96] fixed smart contract results processor --- .../smartContractResultsProcessor.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 2f3c7112..843a58c2 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -45,18 +45,22 @@ func (proc *smartContractResultsProcessor) processSCRs( numOfShards uint32, ) []*indexerData.ScResult { allSCRs := make([]*indexerData.ScResult, 0, len(txsHandler)) + + // we need this map because proc.processSCRsFromMiniblock removes items + // from the map in order to remain with the smart contract results that have no transaction on the current shard + workingSCRSMap := copySCRSMap(txsHandler) for _, mb := range body.MiniBlocks { if mb.Type != block.SmartContractResultBlock { continue } - indexerSCRs := proc.processSCRsFromMiniblock(header, mb, txsHandler, numOfShards) + indexerSCRs := proc.processSCRsFromMiniblock(header, mb, workingSCRSMap, numOfShards) allSCRs = append(allSCRs, indexerSCRs...) } selfShardID := header.GetShardID() - for scrHash, noMBScr := range txsHandler { + for scrHash, noMBScr := range workingSCRSMap { scr, ok := noMBScr.GetTxHandler().(*smartContractResult.SmartContractResult) if !ok { continue @@ -171,3 +175,11 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( InitialTxGasUsed: initialTxGasUsed, } } + +func copySCRSMap(initial map[string]data.TransactionHandlerWithGasUsedAndFee) map[string]data.TransactionHandlerWithGasUsedAndFee { + newMap := make(map[string]data.TransactionHandlerWithGasUsedAndFee) + for key, value := range initial { + newMap[key] = value + } + return newMap +} From de67b59922ffcdf77ee0239cd55e9987d57d2817 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 9 Nov 2022 12:01:38 +0200 Subject: [PATCH 95/96] fix com --- .../elasticproc/transactions/smartContractResultsProcessor.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 843a58c2..abbb2a0b 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -46,8 +46,7 @@ func (proc *smartContractResultsProcessor) processSCRs( ) []*indexerData.ScResult { allSCRs := make([]*indexerData.ScResult, 0, len(txsHandler)) - // we need this map because proc.processSCRsFromMiniblock removes items - // from the map in order to remain with the smart contract results that have no transaction on the current shard + // a copy of the SCRS map is needed because proc.processSCRsFromMiniblock would remove items from the original map workingSCRSMap := copySCRSMap(txsHandler) for _, mb := range body.MiniBlocks { if mb.Type != block.SmartContractResultBlock { From 264cd0d60ee327477a0fc71dfdb94fffcaf03645 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 11 Nov 2022 11:25:30 +0200 Subject: [PATCH 96/96] proper tags --- go.mod | 4 ++-- go.sum | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 2bef6733..f706054e 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108113024-38dd797ed574 + github.com/ElrondNetwork/elrond-go-core v1.1.24 github.com/ElrondNetwork/elrond-go-logger v1.0.8 - github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 + github.com/ElrondNetwork/elrond-vm-common v1.3.25 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 1af5bf60..37c080af 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= -github.com/ElrondNetwork/elrond-go-core v1.1.20-0.20220915145036-720d0233becd/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= -github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108113024-38dd797ed574 h1:XCj1Fo6Xl8EzAFhaIFcCY393Cu2HPppAKeOdxMaTFT4= -github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108113024-38dd797ed574/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.24 h1:jamx50C1dP50uwh11EXT0+4pq0E0G3YYTK2xKEewDEk= +github.com/ElrondNetwork/elrond-go-core v1.1.24/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= github.com/ElrondNetwork/elrond-go-logger v1.0.8/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= -github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6 h1:LM7cObmyt3ZyGXGxE8y5K/ME4SDO/ZuSaixQpsjWEYM= -github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6/go.mod h1:7qdwuZaUr5JFiOABCVgAgiuMHEJ2B8Lv5cTTAFedZNA= +github.com/ElrondNetwork/elrond-vm-common v1.3.25 h1:Q8VpD8vfgWS2odnwiTULz1beeUd6pLsSL3s+uKKV/3M= +github.com/ElrondNetwork/elrond-vm-common v1.3.25/go.mod h1:PaQtmmGpxi3flbCAyvY6LdTVPe+I1JyfpZZp+EKGtUc= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=