indexer binary

.github/workflows/golangci-lint.yml

@@ -0,0 +1,37 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches: [ master, development, feat/*, rc/* ]
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: golangci linter
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.17.6
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.45.2
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          args: --timeout 10m0s --max-issues-per-linter 0 --max-same-issues 0 --print-issued-lines
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          only-new-issues: true
+
+          # Optional: if set to true then the action will use pre-installed Go
+          # skip-go-installation: true

.github/workflows/pr-build.yml

@@ -29,4 +29,6 @@ jobs:
               dep ensure
           fi
       - name: Build
-        run: go build
+        run: |
+          cd cmd/elasticindexer 
+          go build

client/elasticClient.go

@@ -8,8 +8,8 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/data"
+	"github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	logger "github.com/ElrondNetwork/elrond-go-logger"
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
@@ -41,7 +41,7 @@ type elasticClient struct {
 // NewElasticClient will create a new instance of elasticClient
 func NewElasticClient(cfg elasticsearch.Config) (*elasticClient, error) {
 	if len(cfg.Addresses) == 0 {
-		return nil, indexer.ErrNoElasticUrlProvided
+		return nil, dataindexer.ErrNoElasticUrlProvided
 	}
 
 	es, err := elasticsearch.NewClient(cfg)
@@ -302,7 +302,7 @@ func (ec *elasticClient) createPolicy(policyName string, policy *bytes.Buffer) e
 
 	errStr := fmt.Sprintf("%v", existsRes.Error)
 	if existsRes.Status == http.StatusConflict && !strings.Contains(errStr, errPolicyAlreadyExists) {
-		return indexer.ErrCouldNotCreatePolicy
+		return dataindexer.ErrCouldNotCreatePolicy
 	}
 
 	return nil

client/elasticClientCommon.go

@@ -10,8 +10,8 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/data"
+	"github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
 )
 
@@ -67,7 +67,7 @@ func elasticDefaultErrorResponseHandler(res *esapi.Response) error {
 		isBackOffError := strings.Contains(string(bodyBytes), fmt.Sprintf("%d", http.StatusForbidden)) ||
 			strings.Contains(string(bodyBytes), fmt.Sprintf("%d", http.StatusTooManyRequests))
 		if isBackOffError {
-			errToReturn = indexer.ErrBackOff
+			errToReturn = dataindexer.ErrBackOff
 		}
 
 		return fmt.Errorf("%w, cannot unmarshal elastic response body to map[string]interface{}, "+
@@ -213,7 +213,7 @@ func kibanaResponseErrorHandler(res *esapi.Response) error {
 	log.Warn("elasticClient.parseResponse",
 		"error returned by elastic API", errorRes.Error,
 		"code", res.StatusCode)
-	return indexer.ErrBackOff
+	return dataindexer.ErrBackOff
 }
 
 func newRequest(method, path string, body *bytes.Buffer) *http.Request {
@@ -263,7 +263,7 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE
 	if err != nil {
 		log.Warn("elasticClient.parseResponse",
 			"could not load response body:", err.Error())
-		return indexer.ErrBackOff
+		return dataindexer.ErrBackOff
 	}
 
 	return nil

client/elasticClientCommon_test.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/mock"
+	"github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +55,7 @@ func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpForbiddenError
 	resp := createMockEsapiResponseWithText(httpErrString)
 	err := elasticDefaultErrorResponseHandler(resp)
 
-	assert.True(t, errorsGo.Is(err, indexer.ErrBackOff))
+	assert.True(t, errorsGo.Is(err, dataindexer.ErrBackOff))
 }
 
 func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpTooManyRequestsErrorShouldSignalBackOffErr(t *testing.T) {
@@ -65,7 +65,7 @@ func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithHttpTooManyRequest
 	resp := createMockEsapiResponseWithText(httpErrString)
 	err := elasticDefaultErrorResponseHandler(resp)
 
-	assert.True(t, errorsGo.Is(err, indexer.ErrBackOff))
+	assert.True(t, errorsGo.Is(err, dataindexer.ErrBackOff))
 }
 
 func TestElasticDefaultErrorResponseHandler_UnmarshalFailsWithGenericError(t *testing.T) {

client/elasticClient_test.go

@@ -7,9 +7,9 @@ import (
 	"os"
 	"testing"
 
-	indexer "github.com/ElrondNetwork/elastic-indexer-go"
 	"github.com/ElrondNetwork/elastic-indexer-go/client/logging"
 	"github.com/ElrondNetwork/elastic-indexer-go/data"
+	indexer "github.com/ElrondNetwork/elastic-indexer-go/process/dataindexer"
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/stretchr/testify/require"
 )

cmd/elasticindexer/config/config.toml

@@ -0,0 +1,26 @@
+[config]
+    enabled-indices =  [
+        "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory",
+        "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags",
+        "logs", "delegators", "operations", "collections"
+    ]
+    [config.web-socket]
+        server-url = "localhost:22111"
+        data-marshaller-type = "json"
+    [config.address-converter]
+        length = 32
+        type = "bech32"
+    [config.validator-keys-converter]
+        length = 96
+        type = "hex"
+    [config.hasher]
+        type = "blake2b"
+    [config.marshaller]
+        type = "gogo protobuf"
+    [config.economics]
+        denomination = 18
+    [config.logs]
+        log-file-life-span-in-mb = 1024 # 1GB
+        log-file-life-span-in-sec = 432000 # 5 days
+        log-file-prefix = "elastic-indexer"
+        logs-path = "logs"

cmd/elasticindexer/config/prefs.toml

@@ -0,0 +1,6 @@
+[elastic-cluster]
+    use-kibana = false
+    url = "http://localhost:9200"
+    username = ""
+    password = ""
+    bulk-request-max-size-in-bytes = 4194304 # 4MB

cmd/elasticindexer/flags.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	logger "github.com/ElrondNetwork/elrond-go-logger"
+	"github.com/urfave/cli"
+)
+
+var (
+	configurationFile = cli.StringFlag{
+		Name:  "config",
+		Usage: "The main configuration file to load",
+		Value: "./config/config.toml",
+	}
+	// configurationPreferencesFile defines a flag for the path to the preferences toml configuration file
+	configurationPreferencesFile = cli.StringFlag{
+		Name: "config-preferences",
+		Usage: "The [path] for the preferences configuration file. This TOML file contains " +
+			"preferences configurations, such as the node display name or the shard to start in when starting as observer",
+		Value: "./config/prefs.toml",
+	}
+	logLevel = cli.StringFlag{
+		Name: "log-level",
+		Usage: "This flag specifies the logger `level(s)`. It can contain multiple comma-separated value. For example" +
+			", if set to *:INFO the logs for all packages will have the INFO level. However, if set to *:INFO,api:DEBUG" +
+			" the logs for all packages will have the INFO level, excepting the api package which will receive a DEBUG" +
+			" log level.",
+		Value: "*:" + logger.LogInfo.String(),
+	}
+	logSaveFile = cli.BoolFlag{
+		Name:  "log-save",
+		Usage: "Boolean option for enabling log saving. If set, it will automatically save all the logs into a file.",
+	}
+)

cmd/elasticindexer/main.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/ElrondNetwork/elastic-indexer-go/config"
+	"github.com/ElrondNetwork/elastic-indexer-go/factory"
+	"github.com/ElrondNetwork/elrond-go-core/core"
+	"github.com/ElrondNetwork/elrond-go-core/core/check"
+	"github.com/ElrondNetwork/elrond-go-core/core/closing"
+	logger "github.com/ElrondNetwork/elrond-go-logger"
+	"github.com/ElrondNetwork/elrond-go-logger/file"
+	"github.com/urfave/cli"
+)
+
+var (
+	log          = logger.GetOrCreate("indexer")
+	helpTemplate = `NAME:
+   {{.Name}} - {{.Usage}}
+USAGE:
+   {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}
+   {{if len .Authors}}
+AUTHOR:
+   {{range .Authors}}{{ . }}{{end}}
+   {{end}}{{if .Commands}}
+GLOBAL OPTIONS:
+   {{range .VisibleFlags}}{{.}}
+   {{end}}
+VERSION:
+   {{.Version}}
+   {{end}}
+`
+)
+
+func main() {
+	app := cli.NewApp()
+	cli.AppHelpTemplate = helpTemplate
+	app.Name = "Elastic indexer"
+	app.Usage = "This tool will index data in an Elasticsearch database"
+	app.Flags = []cli.Flag{
+		configurationFile,
+		configurationPreferencesFile,
+		logLevel,
+		logSaveFile,
+	}
+	app.Authors = []cli.Author{
+		{
+			Name:  "The Elrond Team",
+			Email: "[email protected]",
+		},
+	}
+
+	app.Action = startIndexer
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Error(err.Error())
+		os.Exit(1)
+	}
+}
+
+func startIndexer(ctx *cli.Context) error {
+	cfg, err := loadMainConfig(ctx.GlobalString(configurationFile.Name))
+	if err != nil {
+		return err
+	}
+
+	clusterCfg, err := loadClusterConfig(ctx.GlobalString(configurationPreferencesFile.Name))
+	if err != nil {
+		return err
+	}
+
+	fileLogging, err := initializeLogger(ctx, cfg)
+	if err != nil {
+		return err
+	}
+
+	wsClient, err := factory.CreateWsIndexer(cfg, clusterCfg)
+	if err != nil {
+		log.Error("cannot create ws indexer", "error", err)
+	}
+
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+
+	go wsClient.Start()
+
+	<-interrupt
+	wsClient.Close()
+	if !check.IfNilReflect(fileLogging) {
+		err = fileLogging.Close()
+		log.LogIfError(err)
+	}
+	return nil
+}
+
+func loadMainConfig(filepath string) (config.Config, error) {
+	cfg := config.Config{}
+	err := core.LoadTomlFile(&cfg, filepath)
+
+	return cfg, err
+}
+
+func loadClusterConfig(filepath string) (config.ClusterConfig, error) {
+	cfg := config.ClusterConfig{}
+	err := core.LoadTomlFile(&cfg, filepath)
+
+	return cfg, err
+}
+
+func initializeLogger(ctx *cli.Context, cfg config.Config) (closing.Closer, error) {
+	logLevelFlagValue := ctx.GlobalString(logLevel.Name)
+	err := logger.SetLogLevel(logLevelFlagValue)
+	if err != nil {
+		return nil, err
+	}
+
+	withLogFile := ctx.GlobalBool(logSaveFile.Name)
+	if !withLogFile {
+		return nil, nil
+	}
+
+	workingDir, err := os.Getwd()
+	if err != nil {
+		log.LogIfError(err)
+		workingDir = ""
+	}
+
+	fileLogging, err := file.NewFileLogging(file.ArgsFileLogging{
+		WorkingDir:      workingDir,
+		DefaultLogsPath: cfg.Config.Logs.LogsPath,
+		LogFilePrefix:   cfg.Config.Logs.LogFilePrefix,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("%w creating a log file", err)
+	}
+
+	err = fileLogging.ChangeFileLifeSpan(
+		time.Second*time.Duration(cfg.Config.Logs.LogFileLifeSpanInSec),
+		uint64(cfg.Config.Logs.LogFileLifeSpanInMB),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	err = logger.RemoveLogObserver(os.Stdout)
+	if err != nil {
+		return nil, err
+	}
+
+	err = logger.AddLogObserver(os.Stdout, &logger.PlainFormatter{})
+	if err != nil {
+		return nil, err
+	}
+
+	return fileLogging, nil
+}