Merge rc v1 7 next into sovereign 14 may 2024

client/elasticClient.go

@@ -153,8 +153,14 @@ func (ec *elasticClient) DoQueryRemove(ctx context.Context, index string, body *
 		log.Warn("elasticClient.doRefresh", "cannot do refresh", err)
 	}
 
+	writeIndex, err := ec.getWriteIndex(index)
+	if err != nil {
+		log.Warn("elasticClient.getWriteIndex", "cannot do get write index", err)
+		return err
+	}
+
 	res, err := ec.client.DeleteByQuery(
-		[]string{index},
+		[]string{writeIndex},
 		body,
 		ec.client.DeleteByQuery.WithIgnoreUnavailable(true),
 		ec.client.DeleteByQuery.WithConflicts(esConflictsPolicy),
@@ -323,6 +329,39 @@ func (ec *elasticClient) createAlias(alias string, index string) error {
 	return parseResponse(res, nil, elasticDefaultErrorResponseHandler)
 }
 
+func (ec *elasticClient) getWriteIndex(alias string) (string, error) {
+	res, err := ec.client.Indices.GetAlias(
+		ec.client.Indices.GetAlias.WithIndex(alias),
+	)
+	if err != nil {
+		return "", err
+	}
+
+	var indexData map[string]struct {
+		Aliases map[string]struct {
+			IsWriteIndex bool `json:"is_write_index"`
+		} `json:"aliases"`
+	}
+	err = parseResponse(res, &indexData, elasticDefaultErrorResponseHandler)
+	if err != nil {
+		return "", err
+	}
+
+	for index, details := range indexData {
+		if len(indexData) == 1 {
+			return index, nil
+		}
+
+		for _, indexAlias := range details.Aliases {
+			if indexAlias.IsWriteIndex {
+				return index, nil
+			}
+		}
+	}
+
+	return alias, nil
+}
+
 // UpdateByQuery will update all the documents that match the provided query from the provided index
 func (ec *elasticClient) UpdateByQuery(ctx context.Context, index string, buff *bytes.Buffer) error {
 	reader := bytes.NewReader(buff.Bytes())

client/elasticClientCommon.go

@@ -96,7 +96,7 @@ func elasticBulkRequestResponseHandler(res *esapi.Response) error {
 		return fmt.Errorf("%s", res.String())
 	}
 
-	bodyBytes, err := ioutil.ReadAll(res.Body)
+	bodyBytes, err := io.ReadAll(res.Body)
 	if err != nil {
 		return fmt.Errorf("%w cannot read elastic response body bytes", err)
 	}

client/elasticClient_test.go

@@ -2,7 +2,7 @@ package client
 
 import (
 	"context"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -53,7 +53,7 @@ func TestElasticClient_DoMultiGet(t *testing.T) {
 		jsonFile, err := os.Open("./testsData/response-multi-get.json")
 		require.Nil(t, err)
 
-		byteValue, _ := ioutil.ReadAll(jsonFile)
+		byteValue, _ := io.ReadAll(jsonFile)
 		_, _ = w.Write(byteValue)
 	}
 
@@ -75,3 +75,51 @@ func TestElasticClient_DoMultiGet(t *testing.T) {
 	_, ok := resMap["docs"]
 	require.True(t, ok)
 }
+
+func TestElasticClient_GetWriteIndexMultipleIndicesBehind(t *testing.T) {
+	handler := http.NotFound
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler(w, r)
+	}))
+	defer ts.Close()
+
+	handler = func(w http.ResponseWriter, r *http.Request) {
+		jsonFile, err := os.Open("./testsData/response-get-alias.json")
+		require.Nil(t, err)
+
+		byteValue, _ := io.ReadAll(jsonFile)
+		_, _ = w.Write(byteValue)
+	}
+
+	esClient, _ := NewElasticClient(elasticsearch.Config{
+		Addresses: []string{ts.URL},
+		Logger:    &logging.CustomLogger{},
+	})
+	res, err := esClient.getWriteIndex("blocks")
+	require.Nil(t, err)
+	require.Equal(t, "blocks-000004", res)
+}
+
+func TestElasticClient_GetWriteIndexOneIndex(t *testing.T) {
+	handler := http.NotFound
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler(w, r)
+	}))
+	defer ts.Close()
+
+	handler = func(w http.ResponseWriter, r *http.Request) {
+		jsonFile, err := os.Open("./testsData/response-get-alias-only-one-index.json")
+		require.Nil(t, err)
+
+		byteValue, _ := io.ReadAll(jsonFile)
+		_, _ = w.Write(byteValue)
+	}
+
+	esClient, _ := NewElasticClient(elasticsearch.Config{
+		Addresses: []string{ts.URL},
+		Logger:    &logging.CustomLogger{},
+	})
+	res, err := esClient.getWriteIndex("delegators")
+	require.Nil(t, err)
+	require.Equal(t, "delegators-000001", res)
+}

client/testsData/response-get-alias-only-one-index.json

@@ -0,0 +1,7 @@
+{
+  "delegators-000001" : {
+    "aliases" : {
+      "delegators" : { }
+    }
+  }
+}

client/testsData/response-get-alias.json

@@ -0,0 +1,30 @@
+{
+  "blocks-000003": {
+    "aliases": {
+      "blocks": {
+        "is_write_index": false
+      }
+    }
+  },
+  "blocks-000004": {
+    "aliases": {
+      "blocks": {
+        "is_write_index": true
+      }
+    }
+  },
+  "blocks-000002": {
+    "aliases": {
+      "blocks": {
+        "is_write_index": false
+      }
+    }
+  },
+  "blocks-000001": {
+    "aliases": {
+      "blocks": {
+        "is_write_index": false
+      }
+    }
+  }
+}

cmd/elasticindexer/config/config.toml

@@ -2,7 +2,7 @@
     available-indices =  [
         "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory",
         "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags",
-        "logs", "delegators", "operations", "esdts"
+        "logs", "delegators", "operations", "esdts", "values", "events"
     ]
     [config.address-converter]
         length = 32

cmd/elasticindexer/flags.go

@@ -47,10 +47,4 @@ var (
 		Name:  "disable-ansi-color",
 		Usage: "Boolean option for disabling ANSI colors in the logging system.",
 	}
-	importDB = cli.BoolFlag{
-		Name: "import-db",
-		Usage: "This flag, when enabled, triggers the indexer to operate in import database mode. In this mode," +
-			" the indexer excludes the indexing of cross shard transactions received from the source shard. " +
-			"This flag must be enabled when the observers are in import database mode.",
-	}
 )

cmd/elasticindexer/main.go

@@ -39,6 +39,18 @@ VERSION:
 `
 )
 
+// appVersion should be populated at build time using ldflags
+// Usage examples:
+// linux/mac:
+//
+//	go build -v -ldflags="-X main.appVersion=$(git describe --tags --long --dirty)"
+//
+// windows:
+//
+//	for /f %i in ('git describe --tags --long --dirty') do set VERS=%i
+//	go build -v -ldflags="-X main.version=%VERS%"
+var version = "undefined"
+
 func main() {
 	app := cli.NewApp()
 	cli.AppHelpTemplate = helpTemplate
@@ -51,7 +63,6 @@ func main() {
 		logLevel,
 		logSaveFile,
 		disableAnsiColor,
-		importDB,
 	}
 	app.Authors = []cli.Author{
 		{
@@ -60,6 +71,7 @@ func main() {
 		},
 	}
 
+	app.Version = version
 	app.Action = startIndexer
 
 	err := app.Run(os.Args)
@@ -85,9 +97,8 @@ func startIndexer(ctx *cli.Context) error {
 		return fmt.Errorf("%w while initializing the logger", err)
 	}
 
-	importDBMode := ctx.GlobalBool(importDB.Name)
 	statusMetrics := metrics.NewStatusMetrics()
-	wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, importDBMode, statusMetrics)
+	wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, statusMetrics, ctx.App.Version)
 	if err != nil {
 		return fmt.Errorf("%w while creating the indexer", err)
 	}

data/data.go

@@ -47,3 +47,9 @@ type ResponseScroll struct {
 		} `json:"hits"`
 	} `json:"hits"`
 }
+
+// KeyValueObj is the dto for values index
+type KeyValueObj struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}

data/event.go

@@ -0,0 +1,20 @@
+package data
+
+import "time"
+
+// LogEvent is the dto for the log event structure
+type LogEvent struct {
+	ID             string        `json:"-"`
+	TxHash         string        `json:"txHash"`
+	OriginalTxHash string        `json:"originalTxHash,omitempty"`
+	LogAddress     string        `json:"logAddress"`
+	Address        string        `json:"address"`
+	Identifier     string        `json:"identifier"`
+	Data           string        `json:"data,omitempty"`
+	AdditionalData []string      `json:"additionalData,omitempty"`
+	Topics         []string      `json:"topics"`
+	Order          int           `json:"order"`
+	TxOrder        int           `json:"txOrder"`
+	ShardID        uint32        `json:"shardID"`
+	Timestamp      time.Duration `json:"timestamp,omitempty"`
+}

data/logs.go

@@ -37,4 +37,6 @@ type PreparedLogsResults struct {
 	TokensInfo              []*TokenInfo
 	NFTsDataUpdates         []*NFTDataUpdate
 	TokenRolesAndProperties *tokeninfo.TokenRolesAndProperties
+	DBLogs                  []*Logs
+	DBEvents                []*LogEvent
 }

data/scresult.go

@@ -39,6 +39,7 @@ type ScResult struct {
 	CanBeIgnored       bool          `json:"canBeIgnored,omitempty"`
 	OriginalSender     string        `json:"originalSender,omitempty"`
 	HasLogs            bool          `json:"hasLogs,omitempty"`
+	ExecutionOrder     int           `json:"-"`
 	SenderAddressBytes []byte        `json:"-"`
 	InitialTxGasUsed   uint64        `json:"-"`
 	InitialTxFee       string        `json:"-"`

data/transaction.go

@@ -5,8 +5,8 @@ import (
 )
 
 // Transaction is a structure containing all the fields that need
-//  to be saved for a transaction. It has all the default fields
-//  plus some extra information for ease of search and filter
+// to be saved for a transaction. It has all the default fields
+// plus some extra information for ease of search and filter
 type Transaction struct {
 	MBHash               string        `json:"miniBlockHash"`
 	Nonce                uint64        `json:"nonce"`
@@ -48,6 +48,7 @@ type Transaction struct {
 	GuardianSignature    string        `json:"guardianSignature,omitempty"`
 	ErrorEvent           bool          `json:"errorEvent,omitempty"`
 	CompletedEvent       bool          `json:"completedEvent,omitempty"`
+	ExecutionOrder       int           `json:"-"`
 	SmartContractResults []*ScResult   `json:"-"`
 	Hash                 string        `json:"-"`
 	BlockHash            string        `json:"-"`

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.0"
 services:
   elasticsearch:
     container_name: es-container

factory/wsIndexerFactory.go

@@ -17,13 +17,13 @@ import (
 var log = logger.GetOrCreate("elasticindexer")
 
 // CreateWsIndexer will create a new instance of wsindexer.WSClient
-func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, importDB bool, statusMetrics core.StatusMetricsHandler) (wsindexer.WSClient, error) {
+func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, statusMetrics core.StatusMetricsHandler, version string) (wsindexer.WSClient, error) {
 	wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType)
 	if err != nil {
 		return nil, err
 	}
 
-	dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, importDB, statusMetrics)
+	dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, statusMetrics, version)
 	if err != nil {
 		return nil, err
 	}
@@ -55,8 +55,8 @@ func createDataIndexer(
 	cfg config.Config,
 	clusterCfg config.ClusterConfig,
 	wsMarshaller marshal.Marshalizer,
-	importDB bool,
 	statusMetrics core.StatusMetricsHandler,
+	version string,
 ) (wsindexer.DataIndexer, error) {
 	marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type)
 	if err != nil {
@@ -88,8 +88,8 @@ func createDataIndexer(
 		AddressPubkeyConverter:   addressPubkeyConverter,
 		ValidatorPubkeyConverter: validatorPubkeyConverter,
 		HeaderMarshaller:         wsMarshaller,
-		ImportDB:                 importDB,
 		StatusMetrics:            statusMetrics,
+		Version:                  version,
 	})
 }
 

go.mod

@@ -6,10 +6,10 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.12.0
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-gonic/gin v1.9.1
-	github.com/multiversx/mx-chain-communication-go v1.0.12
-	github.com/multiversx/mx-chain-core-go v1.2.19-0.20240118100536-661f5af64039
-	github.com/multiversx/mx-chain-logger-go v1.0.14-0.20231129101244-c44fa1c79b03
-	github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20240118100602-3d0d315083e8
+	github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e
+	github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784
+	github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57
+	github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa
 	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.37.0
 	github.com/stretchr/testify v1.8.4

go.sum

@@ -247,15 +247,15 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
-github.com/multiversx/mx-chain-communication-go v1.0.12 h1:67WOaf87gpwouydD1AAOHw5LMGZh7NfITrp/KqFY3Tw=
-github.com/multiversx/mx-chain-communication-go v1.0.12/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4=
-github.com/multiversx/mx-chain-core-go v1.2.19-0.20240118100536-661f5af64039 h1:rusWamF1HWGAHsmdz/5TSfxHpe36HHvXCEKz3tic2yo=
-github.com/multiversx/mx-chain-core-go v1.2.19-0.20240118100536-661f5af64039/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0=
-github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU=
-github.com/multiversx/mx-chain-logger-go v1.0.14-0.20231129101244-c44fa1c79b03 h1:krjJTyN9jrFTK0goMGFdgvJGy6bYSqe8EtI/HCceUmU=
-github.com/multiversx/mx-chain-logger-go v1.0.14-0.20231129101244-c44fa1c79b03/go.mod h1:fH/fR/GEBsDjPkBoZDVJMoYo2HhlA7++DP6QfITJ1N8=
-github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20240118100602-3d0d315083e8 h1:0/k3n7Ak66oU1ygy8XR+4Q53DGmhS0VrMdKcZO433FI=
-github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20240118100602-3d0d315083e8/go.mod h1:1ZUnRk7l/eTOyu2DOxy6zfEn1SAM/1u0nHUXE1Jw9xY=
+github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e h1:Tsmwhu+UleE+l3buPuqXSKTqfu5FbPmzQ4MjMoUvCWA=
+github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e/go.mod h1:2yXl18wUbuV3cRZr7VHxM1xo73kTaC1WUcu2kx8R034=
+github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784 h1:Dxm8MIIbEQW9hUIjfiVFm7mR+UbOSkb7xhMtXHP7dmk=
+github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0=
+github.com/multiversx/mx-chain-crypto-go v1.2.12-0.20240508074452-cc21c1b505df h1:clihfi78bMEOWk/qw6WA4uQbCM2e2NGliqswLAvw19k=
+github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 h1:g9t410dqjcb7UUptbVd/H6Ua12sEzWU4v7VplyNvRZ0=
+github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57/go.mod h1:cY6CIXpndW5g5PTPn4WzPwka/UBEf+mgw+PSY5pHGAU=
+github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa h1:CJyBXV0PHDE3/bjJU3quoNBjOmGjrn/EBqU7wBdcqhk=
+github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa/go.mod h1:/scqfBjA912V3Z2sPlkRVxWAE3puVNS2adQegm4LjCc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=

integrationtests/accountsBalanceNftTransfer_test.go

@@ -34,6 +34,7 @@ func createOutportBlockWithHeader(
 			TransactionPool: pool,
 			AlteredAccounts: coreAlteredAccounts,
 			NumberOfShards:  numOfShards,
+			ShardID:         header.GetShardID(),
 		},
 		Header: header,
 	}