Merge rc 1.5 in 1.6

data/consts.go

@@ -6,4 +6,6 @@ const (
 
 	// MaxESDTValueLength defines the maximum length for an ESDT value that can be parsed
 	MaxESDTValueLength = 100
+	// MaxFieldLength defines the maximum length for a keyword field, approximating the maximum length of the keyword type.
+	MaxFieldLength = 30000
 )

process/elasticproc/accounts/accountsProcessor_test.go

@@ -177,9 +177,10 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) {
 	require.Equal(t, big.NewInt(1), balance)
 	require.Equal(t, "6f6b", prop)
 	require.Equal(t, &data.TokenMetaData{
-		Name:      nftName,
-		Creator:   creator,
-		Royalties: 2,
+		Name:       nftName,
+		Creator:    creator,
+		Royalties:  2,
+		Attributes: make([]byte, 0),
 	}, metaData)
 }
 
@@ -343,7 +344,8 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) {
 		Properties:      "3032",
 		TokenNonce:      15,
 		Data: &data.TokenMetaData{
-			Creator: "creator",
+			Creator:    "creator",
+			Attributes: make([]byte, 0),
 		},
 		Timestamp: time.Duration(123),
 	}, res[hex.EncodeToString([]byte(addr))+"-token-15"])
@@ -357,7 +359,8 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) {
 		Properties:      "3032",
 		TokenNonce:      16,
 		Data: &data.TokenMetaData{
-			Creator: "creator",
+			Creator:    "creator",
+			Attributes: make([]byte, 0),
 		},
 		Timestamp: time.Duration(123),
 	}, res[hex.EncodeToString([]byte(addr))+"-token-16"])

process/elasticproc/converters/field.go

@@ -0,0 +1,22 @@
+package converters
+
+import "github.com/multiversx/mx-chain-es-indexer-go/data"
+
+// TruncateFieldIfExceedsMaxLength will truncate the provided field if the max length exceeds
+func TruncateFieldIfExceedsMaxLength(field string) string {
+	if len(field) > data.MaxFieldLength {
+		return field[:data.MaxFieldLength]
+	}
+
+	return field
+}
+
+//TruncateSliceElementsIfExceedsMaxLength will truncate the provided slice of the field if the max length is exceeded
+func TruncateSliceElementsIfExceedsMaxLength(fields []string) []string {
+	var localFields []string
+	for _, field := range fields {
+		localFields = append(localFields, TruncateFieldIfExceedsMaxLength(field))
+	}
+
+	return localFields
+}

process/elasticproc/converters/field_test.go

@@ -0,0 +1,34 @@
+package converters
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/multiversx/mx-chain-es-indexer-go/data"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTruncateFieldIfExceedsMaxLength(t *testing.T) {
+	t.Parallel()
+
+	field := "my-field"
+	truncated := TruncateFieldIfExceedsMaxLength(field)
+	require.Equal(t, field, truncated)
+
+	maxLengthField := strings.Repeat("a", data.MaxFieldLength) + strings.Repeat("b", data.MaxFieldLength)
+	truncated = TruncateFieldIfExceedsMaxLength(maxLengthField)
+	require.Equal(t, strings.Repeat("a", data.MaxFieldLength), truncated)
+}
+
+func TestTruncateSliceElementsIfExceedsMaxLength(t *testing.T) {
+	t.Parallel()
+
+	fields := []string{"my-field", "my-field"}
+	truncated := TruncateSliceElementsIfExceedsMaxLength(fields)
+	require.Equal(t, fields, truncated)
+
+	bigField := strings.Repeat("a", data.MaxFieldLength) + strings.Repeat("b", data.MaxFieldLength)
+	fields = []string{bigField, bigField}
+	truncated = TruncateSliceElementsIfExceedsMaxLength(fields)
+	require.Equal(t, []string{strings.Repeat("a", data.MaxFieldLength), strings.Repeat("a", data.MaxFieldLength)}, truncated)
+}

process/elasticproc/converters/tags.go

@@ -24,7 +24,7 @@ func ExtractMetaDataFromAttributes(attributes []byte) string {
 		return ""
 	}
 
-	return res[0]
+	return TruncateFieldIfExceedsMaxLength(res[0])
 }
 
 func extractFromAttributes(attributes []byte, key string) []string {

process/elasticproc/converters/tokenMetaData.go

@@ -25,14 +25,22 @@ func PrepareTokenMetaData(tokenMetadata *alteredAccount.TokenMetaData) *data.Tok
 		return nil
 	}
 
+	var uris [][]byte
+	for _, uri := range tokenMetadata.URIs {
+		truncatedURI := TruncateFieldIfExceedsMaxLength(string(uri))
+		uris = append(uris, []byte(truncatedURI))
+	}
+
+	tags := ExtractTagsFromAttributes(tokenMetadata.Attributes)
+	attributes := TruncateFieldIfExceedsMaxLength(string(tokenMetadata.Attributes))
 	return &data.TokenMetaData{
-		Name:               tokenMetadata.Name,
+		Name:               TruncateFieldIfExceedsMaxLength(tokenMetadata.Name),
 		Creator:            tokenMetadata.Creator,
 		Royalties:          tokenMetadata.Royalties,
 		Hash:               tokenMetadata.Hash,
-		URIs:               tokenMetadata.URIs,
-		Attributes:         tokenMetadata.Attributes,
-		Tags:               ExtractTagsFromAttributes(tokenMetadata.Attributes),
+		URIs:               uris,
+		Attributes:         []byte(attributes),
+		Tags:               TruncateSliceElementsIfExceedsMaxLength(tags),
 		MetaData:           ExtractMetaDataFromAttributes(tokenMetadata.Attributes),
 		NonEmptyURIs:       nonEmptyURIs(tokenMetadata.URIs),
 		WhiteListedStorage: whiteListedStorage(tokenMetadata.URIs),

process/elasticproc/logsevents/nftsProcessor_test.go

@@ -50,7 +50,8 @@ func TestNftsProcessor_processLogAndEventsNFTs(t *testing.T) {
 		Issuer:     "",
 		Nonce:      uint64(19),
 		Data: &data.TokenMetaData{
-			Creator: hex.EncodeToString([]byte("creator")),
+			Creator:    hex.EncodeToString([]byte("creator")),
+			Attributes: make([]byte, 0),
 		},
 	}, tokensCreateInfo.GetAll()[0])
 

process/elasticproc/transactions/smartContractResultsProcessor.go

@@ -13,6 +13,7 @@ import (
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	indexerData "github.com/multiversx/mx-chain-es-indexer-go/data"
 	"github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
+	"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters"
 )
 
 type smartContractResultsProcessor struct {
@@ -178,10 +179,10 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult(
 		SenderShard:        senderShard,
 		ReceiverShard:      receiverShard,
 		Operation:          res.Operation,
-		Function:           res.Function,
+		Function:           converters.TruncateFieldIfExceedsMaxLength(res.Function),
 		ESDTValues:         esdtValues,
 		ESDTValuesNum:      esdtValuesNum,
-		Tokens:             res.Tokens,
+		Tokens:             converters.TruncateSliceElementsIfExceedsMaxLength(res.Tokens),
 		Receivers:          receiversAddr,
 		ReceiversShardIDs:  res.ReceiversShardID,
 		IsRelayed:          res.IsRelayed,

process/elasticproc/transactions/transactionDBBuilder.go

@@ -14,6 +14,7 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data/rewardTx"
 	"github.com/multiversx/mx-chain-es-indexer-go/data"
 	"github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
+	"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters"
 )
 
 type dbTransactionBuilder struct {
@@ -84,6 +85,8 @@ func (dtb *dbTransactionBuilder) prepareTransaction(
 		guardianAddress = dtb.addressPubkeyConverter.SilentEncode(tx.GuardianAddr, log)
 	}
 
+	senderUserName := converters.TruncateFieldIfExceedsMaxLength(string(tx.SndUserName))
+	receiverUserName := converters.TruncateFieldIfExceedsMaxLength(string(tx.RcvUserName))
 	return &data.Transaction{
 		Hash:              hex.EncodeToString(txHash),
 		MBHash:            hex.EncodeToString(mbHash),
@@ -105,14 +108,14 @@ func (dtb *dbTransactionBuilder) prepareTransaction(
 		InitialPaidFee:    feeInfo.InitialPaidFee.String(),
 		Fee:               feeInfo.Fee.String(),
 		FeeNum:            feeNum,
-		ReceiverUserName:  tx.RcvUserName,
-		SenderUserName:    tx.SndUserName,
+		ReceiverUserName:  []byte(receiverUserName),
+		SenderUserName:    []byte(senderUserName),
 		IsScCall:          isScCall,
 		Operation:         res.Operation,
-		Function:          res.Function,
+		Function:          converters.TruncateFieldIfExceedsMaxLength(res.Function),
 		ESDTValues:        esdtValues,
 		ESDTValuesNum:     esdtValuesNum,
-		Tokens:            res.Tokens,
+		Tokens:            converters.TruncateSliceElementsIfExceedsMaxLength(res.Tokens),
 		Receivers:         receiversAddr,
 		ReceiversShardIDs: res.ReceiversShardID,
 		IsRelayed:         res.IsRelayed,