feat/altered accounts

.github/workflows/golangci-lint.yml

@@ -0,0 +1,36 @@
+name: golangci-lint
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master, development, feat/*, rc/* ]
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: golangci linter
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.17.6
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.45.2
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          args: --timeout 10m0s --max-issues-per-linter 0 --max-same-issues 0 --print-issued-lines
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          only-new-issues: true
+
+          # Optional: if set to true then the action will use pre-installed Go
+          # skip-go-installation: true

.github/workflows/pr-build.yml

@@ -29,4 +29,6 @@ jobs:
               dep ensure
           fi
       - name: Build
-        run: go build
+        run: |
+          cd cmd/elasticindexer 
+          go build

client/data.go

@@ -18,7 +18,7 @@ type BulkRequestResponse struct {
 	} `json:"items"`
 }
 
-// Item defines the structure of a item from a bulk response
+// Item defines the structure of an item from a bulk response
 type Item struct {
 	Index  string `json:"_index"`
 	ID     string `json:"_id"`

client/elasticClient.go

@@ -8,8 +8,8 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/data"
+	"github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	logger "github.com/ElrondNetwork/elrond-go-logger"
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
@@ -41,7 +41,7 @@ type elasticClient struct {
 // NewElasticClient will create a new instance of elasticClient
 func NewElasticClient(cfg elasticsearch.Config) (*elasticClient, error) {
 	if len(cfg.Addresses) == 0 {
-		return nil, indexer.ErrNoElasticUrlProvided
+		return nil, dataindexer.ErrNoElasticUrlProvided
 	}
 
 	es, err := elasticsearch.NewClient(cfg)
@@ -125,7 +125,7 @@ func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string) error {
 	return elasticBulkRequestResponseHandler(res)
 }
 
-// DoMultiGet wil do a multi get request to elaticsearch server
+// DoMultiGet wil do a multi get request to Elasticsearch server
 func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, resBody interface{}) error {
 	obj := getDocumentsByIDsQuery(ids, withSource)
 	body, err := encode(obj)
@@ -302,15 +302,15 @@ func (ec *elasticClient) createPolicy(policyName string, policy *bytes.Buffer) e
 
 	errStr := fmt.Sprintf("%v", existsRes.Error)
 	if existsRes.Status == http.StatusConflict && !strings.Contains(errStr, errPolicyAlreadyExists) {
-		return indexer.ErrCouldNotCreatePolicy
+		return dataindexer.ErrCouldNotCreatePolicy
 	}
 
 	return nil
 }
 
 // CreateIndexTemplate creates an elasticsearch index template
 func (ec *elasticClient) createIndexTemplate(templateName string, template io.Reader) error {
-	res, err := ec.client.Indices.PutTemplate(templateName, template)
+	res, err := ec.client.Indices.PutTemplate(templateName, template, ec.client.Indices.PutTemplate.WithContext(context.Background()))
 	if err != nil {
 		return err
 	}

client/elasticClientCommon.go

@@ -10,8 +10,8 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/data"
+	"github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
 )
 
@@ -67,7 +67,7 @@ func elasticDefaultErrorResponseHandler(res *esapi.Response) error {
 		isBackOffError := strings.Contains(string(bodyBytes), fmt.Sprintf("%d", http.StatusForbidden)) ||
 			strings.Contains(string(bodyBytes), fmt.Sprintf("%d", http.StatusTooManyRequests))
 		if isBackOffError {
-			errToReturn = indexer.ErrBackOff
+			errToReturn = dataindexer.ErrBackOff
 		}
 
 		return fmt.Errorf("%w, cannot unmarshal elastic response body to map[string]interface{}, "+
@@ -213,7 +213,7 @@ func kibanaResponseErrorHandler(res *esapi.Response) error {
 	log.Warn("elasticClient.parseResponse",
 		"error returned by elastic API", errorRes.Error,
 		"code", res.StatusCode)
-	return indexer.ErrBackOff
+	return dataindexer.ErrBackOff
 }
 
 func newRequest(method, path string, body *bytes.Buffer) *http.Request {
@@ -263,7 +263,7 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE
 	if err != nil {
 		log.Warn("elasticClient.parseResponse",
 			"could not load response body:", err.Error())
-		return indexer.ErrBackOff
+		return dataindexer.ErrBackOff
 	}
 
 	return nil

client/elasticClientCommon_test.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/mock"
+	"github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +55,7 @@ func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpForbiddenError
 	resp := createMockEsapiResponseWithText(httpErrString)
 	err := elasticDefaultErrorResponseHandler(resp)
 
-	assert.True(t, errorsGo.Is(err, indexer.ErrBackOff))
+	assert.True(t, errorsGo.Is(err, dataindexer.ErrBackOff))
 }
 
 func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpTooManyRequestsErrorShouldSignalBackOffErr(t *testing.T) {
@@ -65,7 +65,7 @@ func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpTooManyRequest
 	resp := createMockEsapiResponseWithText(httpErrString)
 	err := elasticDefaultErrorResponseHandler(resp)
 
-	assert.True(t, errorsGo.Is(err, indexer.ErrBackOff))
+	assert.True(t, errorsGo.Is(err, dataindexer.ErrBackOff))
 }
 
 func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithGenericError(t *testing.T) {

client/elasticClient_test.go

@@ -7,9 +7,9 @@ import (
 	"os"
 	"testing"
 
-	indexer "github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/client/logging"
 	"github.com/ElrondNetwork/elastic-indexer-go/data"
+	indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/stretchr/testify/require"
 )

cmd/elasticindexer/config/config.toml

@@ -0,0 +1,23 @@
+[config]
+    available-indices =  [
+        "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory",
+        "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags",
+        "logs", "delegators", "operations", "collections"
+    ]
+    [config.address-converter]
+        length = 32
+        type = "bech32"
+    [config.validator-keys-converter]
+        length = 96
+        type = "hex"
+    [config.hasher]
+        type = "blake2b"
+    [config.marshaller]
+        type = "gogo protobuf"
+    [config.economics]
+        denomination = 18
+    [config.logs]
+        log-file-life-span-in-mb = 1024 # 1GB
+        log-file-life-span-in-sec = 432000 # 5 days
+        log-file-prefix = "elastic-indexer"
+        logs-path = "logs"

cmd/elasticindexer/config/prefs.toml

@@ -0,0 +1,12 @@
+[config]
+    disabled-indices = []
+    [config.web-socket]
+        server-url = "localhost:22111"
+        data-marshaller-type = "json"
+
+    [config.elastic-cluster]
+        use-kibana = false
+        url = "http://localhost:9200"
+        username = ""
+        password = ""
+        bulk-request-max-size-in-bytes = 4194304 # 4MB

cmd/elasticindexer/flags.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	logger "github.com/ElrondNetwork/elrond-go-logger"
+	"github.com/urfave/cli"
+)
+
+var (
+	configurationFile = cli.StringFlag{
+		Name:  "config",
+		Usage: "The main configuration file to load",
+		Value: "./config/config.toml",
+	}
+	// configurationPreferencesFile defines a flag for the path to the preferences toml configuration file
+	configurationPreferencesFile = cli.StringFlag{
+		Name: "config-preferences",
+		Usage: "The [path] for the preferences configuration file. This TOML file contains " +
+			"preferences configurations, such as the node display name or the shard to start in when starting as observer",
+		Value: "./config/prefs.toml",
+	}
+	logLevel = cli.StringFlag{
+		Name: "log-level",
+		Usage: "This flag specifies the logger `level(s)`. It can contain multiple comma-separated value. For example" +
+			", if set to *:INFO the logs for all packages will have the INFO level. However, if set to *:INFO,api:DEBUG" +
+			" the logs for all packages will have the INFO level, excepting the api package which will receive a DEBUG" +
+			" log level.",
+		Value: "*:" + logger.LogInfo.String(),
+	}
+	logSaveFile = cli.BoolFlag{
+		Name:  "log-save",
+		Usage: "Boolean option for enabling log saving. If set, it will automatically save all the logs into a file.",
+	}
+	// disableAnsiColor defines if the logger subsystem should prevent displaying ANSI colors
+	disableAnsiColor = cli.BoolFlag{
+		Name:  "disable-ansi-color",
+		Usage: "Boolean option for disabling ANSI colors in the logging system.",
+	}
+)