Improvement: replace empty string to NULL in database (#785)

* Improvement: replace empty string to NULL in database

* Fix: view

* Some security fixes

cmd/api/handlers/contract.go

@@ -85,7 +85,7 @@ func (ctx *Context) contractPostprocessing(contract contract.Contract) (Contract
 	res.FromModel(contract)
 
 	res.Alias = ctx.CachedAlias(contract.Network, contract.Address)
-	res.DelegateAlias = ctx.CachedAlias(contract.Network, contract.Delegate)
+	res.DelegateAlias = ctx.CachedAlias(contract.Network, contract.Delegate.String())
 
 	if alias, err := ctx.TZIP.Get(contract.Network, contract.Address); err == nil {
 		res.Slug = alias.Slug

cmd/api/handlers/project.go

@@ -106,7 +106,7 @@ func (ctx *Context) GetSimilarContracts(c *gin.Context) {
 		response.Contracts[i].FromModel(similar[i], diff)
 
 		response.Contracts[i].Alias = ctx.CachedAlias(similar[i].Network, similar[i].Address)
-		response.Contracts[i].DelegateAlias = ctx.CachedAlias(similar[i].Network, similar[i].Delegate)
+		response.Contracts[i].DelegateAlias = ctx.CachedAlias(similar[i].Network, similar[i].Delegate.String())
 	}
 
 	c.SecureJSON(http.StatusOK, response)

cmd/api/handlers/responses.go

@@ -84,7 +84,7 @@ func (o *Operation) FromModel(operation operation.Operation) {
 	o.Delegate = operation.Delegate
 	o.Status = operation.Status.String()
 	o.Burned = operation.Burned
-	o.Entrypoint = operation.Entrypoint
+	o.Entrypoint = operation.Entrypoint.String()
 	o.ContentIndex = operation.ContentIndex
 	o.AllocatedDestinationContractBurned = operation.AllocatedDestinationContractBurned
 	o.ConsumedGas = operation.ConsumedGas
@@ -96,12 +96,11 @@ func (o *Operation) FromModel(operation operation.Operation) {
 // ToModel -
 func (o *Operation) ToModel() operation.Operation {
 	return operation.Operation{
-		ID:        o.ID,
-		Hash:      o.Hash,
-		Network:   types.NewNetwork(o.Network),
-		Internal:  o.Internal,
-		Timestamp: o.Timestamp,
-
+		ID:           o.ID,
+		Hash:         o.Hash,
+		Network:      types.NewNetwork(o.Network),
+		Internal:     o.Internal,
+		Timestamp:    o.Timestamp,
 		Level:        o.Level,
 		Kind:         types.NewOperationKind(o.Kind),
 		Source:       o.Source,
@@ -114,8 +113,10 @@ func (o *Operation) ToModel() operation.Operation {
 		Delegate:     o.Delegate,
 		Status:       types.NewOperationStatus(o.Status),
 		Burned:       o.Burned,
-		Entrypoint:   o.Entrypoint,
-
+		Entrypoint: types.NullString{
+			Str:   o.Entrypoint,
+			Valid: o.Entrypoint != "",
+		},
 		AllocatedDestinationContract: o.AllocatedDestinationContract,
 		ConsumedGas:                  o.ConsumedGas,
 		StorageSize:                  o.StorageSize,
@@ -161,15 +162,15 @@ type Contract struct {
 // FromModel -
 func (c *Contract) FromModel(contract contract.Contract) {
 	c.Address = contract.Address
-	c.Delegate = contract.Delegate
+	c.Delegate = contract.Delegate.String()
 	c.Entrypoints = contract.Entrypoints
 	c.Hash = contract.Hash
 	c.Language = contract.Language
 	c.TxCount = contract.TxCount
 	c.LastAction = contract.LastAction
 
 	c.Level = contract.Level
-	c.Manager = contract.Manager
+	c.Manager = contract.Manager.String()
 	c.MigrationsCount = contract.MigrationsCount
 	c.Network = contract.Network.String()
 	c.ProjectID = contract.ProjectID
@@ -470,7 +471,7 @@ func (c *SameContractsResponse) FromModel(same contract.SameResponse, ctx *Conte
 		var contract Contract
 		contract.FromModel(same.Contracts[i])
 		contract.Alias = ctx.CachedAlias(same.Contracts[i].Network, same.Contracts[i].Address)
-		contract.DelegateAlias = ctx.CachedAlias(same.Contracts[i].Network, same.Contracts[i].Delegate)
+		contract.DelegateAlias = ctx.CachedAlias(same.Contracts[i].Network, same.Contracts[i].Delegate.String())
 		c.Contracts[i] = contract
 	}
 }
@@ -535,7 +536,7 @@ func TransferFromModel(model domains.Transfer) (t Transfer) {
 	t.To = model.To
 	t.TokenID = model.TokenID
 	t.Amount = model.Amount.String()
-	t.Parent = model.Parent
+	t.Parent = model.Parent.String()
 	t.Entrypoint = model.Entrypoint
 	t.Hash = model.Hash
 	t.Counter = model.Counter

cmd/api/handlers/tokens.go

@@ -195,10 +195,10 @@ func (ctx *Context) contractToTokens(contracts []contract.Contract, network type
 			Level:         contracts[i].Level,
 			Timestamp:     contracts[i].Timestamp,
 			Address:       contracts[i].Address,
-			Manager:       contracts[i].Manager,
-			Delegate:      contracts[i].Delegate,
+			Manager:       contracts[i].Manager.String(),
+			Delegate:      contracts[i].Delegate.String(),
 			Alias:         ctx.CachedAlias(contracts[i].Network, contracts[i].Address),
-			DelegateAlias: ctx.CachedAlias(contracts[i].Network, contracts[i].Delegate),
+			DelegateAlias: ctx.CachedAlias(contracts[i].Network, contracts[i].Delegate.String()),
 			LastAction:    contracts[i].LastAction,
 			TxCount:       contracts[i].TxCount,
 		}

cmd/indexer/scripts/head_stats_view.sql

@@ -1,5 +1,5 @@
 create materialized view if not exists head_stats AS
-select network, count(*) as value, 'calls_count' as stats_type from operations where entrypoint != '' group by network
+select network, count(*) as value, 'calls_count' as stats_type from operations where entrypoint is not null group by network
 union all
 select network, count(*) as value, 'contracts_count' as stats_type from contracts group by network
 union all

configs/development.yml

@@ -2,36 +2,33 @@ rpc:
   mainnet:
     uri: https://rpc.tzkt.io/mainnet
     timeout: 20
-  florencenet:
-    uri: https://rpc.tzkt.io/florencenobanet
-    timeout: 20
   granadanet:
-    uri: https://granadanet.smartpy.io/
+    uri: https://rpc.tzkt.io/granadanet
     timeout: 20
-  hangzhounet:
-    uri: https://rpc.tzkt.io/hangzhounet
+  hangzhou2net:
+    uri: https://rpc.tzkt.io/hangzhou2net
     timeout: 20
 
 tzkt:
   mainnet:
     uri: https://api.tzkt.io/v1/
     base_uri: https://tzkt.io/
     timeout: 20
-  florencenet:
-    uri: https://api.florencenet.tzkt.io/v1/
-    base_uri: https://florencenet.tzkt.io/
+  granadanet:
+    uri: https://api.granadanet.tzkt.io/v1/
+    base_uri: https://granadanet.tzkt.io/
     timeout: 20
-  hangzhounet:
-    uri: https://api.hangzhounet.tzkt.io/v1/
-    base_uri: https://hangzhounet.tzkt.io/
+  hangzhou2net:
+    uri: https://api.hangzhou2net.tzkt.io/v1/
+    base_uri: https://hangzhou2net.tzkt.io/
     timeout: 20
 
 services:
   mainnet:
     mempool: https://mempool.dipdup.net/v1/graphql
-  florencenet:
+  granadanet:
     mempool: https://mempool.dipdup.net/v1/graphql
-  hangzhounet:
+  hangzhou2net:
     mempool: https://mempool.dipdup.net/v1/graphql
 
 storage:
@@ -72,9 +69,8 @@ api:
     sandbox_mode: false
   networks:
     - mainnet
-    - florencenet
     - granadanet
-    - hangzhounet
+    - hangzhou2net
   pinata:
     key: ${PINATA_KEY}
     secret_key: ${PINATA_SECRET_KEY}
@@ -90,9 +86,8 @@ indexer:
   networks:
     mainnet:
       boost: tzkt
-    florencenet:
     granadanet:
-    hangzhounet:
+    hangzhou2net:
   connections:
     max: 5
     idle: 5
@@ -113,9 +108,8 @@ scripts:
     secret_access_key: ${AWS_SECRET_ACCESS_KEY}
   networks:
     - mainnet
-    - florencenet
     - granadanet
-    - hangzhounet
+    - hangzhou2net
   connections:
     max: 5
     idle: 5

internal/classification/metrics/manager_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/baking-bad/bcdhub/internal/models/contract"
+	"github.com/baking-bad/bcdhub/internal/models/types"
 )
 
 func TestManager_Compute(t *testing.T) {
@@ -21,11 +22,17 @@ func TestManager_Compute(t *testing.T) {
 			name: "Case 1",
 			args: args{
 				a: contract.Contract{
-					Manager: "test",
+					Manager: types.NullString{
+						Str:   "test",
+						Valid: true,
+					},
 					Network: 0,
 				},
 				b: contract.Contract{
-					Manager: "test",
+					Manager: types.NullString{
+						Str:   "test",
+						Valid: true,
+					},
 					Network: 0,
 				},
 			},
@@ -37,11 +44,17 @@ func TestManager_Compute(t *testing.T) {
 			name: "Case 2",
 			args: args{
 				a: contract.Contract{
-					Manager: "other",
+					Manager: types.NullString{
+						Str:   "other",
+						Valid: true,
+					},
 					Network: 1,
 				},
 				b: contract.Contract{
-					Manager: "test",
+					Manager: types.NullString{
+						Str:   "test",
+						Valid: true,
+					},
 					Network: 1,
 				},
 			},
@@ -53,11 +66,17 @@ func TestManager_Compute(t *testing.T) {
 			name: "Case 3",
 			args: args{
 				a: contract.Contract{
-					Manager: "test",
+					Manager: types.NullString{
+						Str:   "test",
+						Valid: true,
+					},
 					Network: 1,
 				},
 				b: contract.Contract{
-					Manager: "test",
+					Manager: types.NullString{
+						Str:   "test",
+						Valid: true,
+					},
 					Network: 2,
 				},
 			},
@@ -69,11 +88,17 @@ func TestManager_Compute(t *testing.T) {
 			name: "Case 4",
 			args: args{
 				a: contract.Contract{
-					Manager: "other",
+					Manager: types.NullString{
+						Str:   "other",
+						Valid: true,
+					},
 					Network: 1,
 				},
 				b: contract.Contract{
-					Manager: "test",
+					Manager: types.NullString{
+						Str:   "test",
+						Valid: true,
+					},
 					Network: 2,
 				},
 			},

internal/fetch/contract.go

@@ -4,17 +4,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
+	"strings"
 
 	"github.com/baking-bad/bcdhub/internal/bcd"
-	"github.com/baking-bad/bcdhub/internal/helpers"
 	"github.com/baking-bad/bcdhub/internal/models/types"
 	"github.com/pkg/errors"
 )
 
-const (
-	contractFormatPath = "%s/contracts/%s/%s_%s.json"
-)
-
 var (
 	delegatorContract = []byte(`[{"prim":"parameter","args":[{"prim":"or","args":[{"prim":"lambda","args":[{"prim":"unit"},{"prim":"list","args":[{"prim":"operation"}]}],"annots":["%do"]},{"prim":"unit","annots":["%default"]}]}]},{"prim":"storage","args":[{"prim":"key_hash"}]},{"prim":"code","args":[[[[{"prim":"DUP"},{"prim":"CAR"},{"prim":"DIP","args":[[{"prim":"CDR"}]]}]],{"prim":"IF_LEFT","args":[[{"prim":"PUSH","args":[{"prim":"mutez"},{"int":"0"}]},{"prim":"AMOUNT"},[[{"prim":"COMPARE"},{"prim":"EQ"}],{"prim":"IF","args":[[],[[{"prim":"UNIT"},{"prim":"FAILWITH"}]]]}],[{"prim":"DIP","args":[[{"prim":"DUP"}]]},{"prim":"SWAP"}],{"prim":"IMPLICIT_ACCOUNT"},{"prim":"ADDRESS"},{"prim":"SENDER"},[[{"prim":"COMPARE"},{"prim":"EQ"}],{"prim":"IF","args":[[],[[{"prim":"UNIT"},{"prim":"FAILWITH"}]]]}],{"prim":"UNIT"},{"prim":"EXEC"},{"prim":"PAIR"}],[{"prim":"DROP"},{"prim":"NIL","args":[{"prim":"operation"}]},{"prim":"PAIR"}]]}]]}]`)
 )
@@ -29,7 +26,10 @@ func RemoveContract(network types.Network, address, protocol, filesDirectory str
 		return err
 	}
 
-	filePath := fmt.Sprintf(contractFormatPath, filesDirectory, network, address, protoSymLink)
+	filePath, err := getFilePath(network, address, protoSymLink, filesDirectory)
+	if err != nil {
+		return err
+	}
 	if _, err = os.Stat(filePath); err == nil {
 		return os.Remove(filePath)
 	} else if !os.IsNotExist(err) {
@@ -39,12 +39,16 @@ func RemoveContract(network types.Network, address, protocol, filesDirectory str
 }
 
 // RemoveAllContracts -
-func RemoveAllContracts(network types.Network, filesDirectory string) error {
+func RemoveAllContracts(network fmt.Stringer, filesDirectory string) error {
 	if filesDirectory == "" {
 		return errors.Errorf("Invalid filesDirectory: %s", filesDirectory)
 	}
 
-	dirPath := fmt.Sprintf("%s/contracts/%s", filesDirectory, network)
+	if err := chechPath(filesDirectory); err != nil {
+		return err
+	}
+
+	dirPath := path.Join(filesDirectory, "contracts", network.String())
 	if _, err := os.Stat(dirPath); err == nil {
 		return os.RemoveAll(dirPath)
 	} else if !os.IsNotExist(err) {
@@ -63,7 +67,10 @@ func Contract(network types.Network, address, protocol, filesDirectory string) (
 		return nil, err
 	}
 
-	filePath := helpers.CleanPath(fmt.Sprintf(contractFormatPath, filesDirectory, network, address, protoSymLink))
+	filePath, err := getFilePath(network, address, protoSymLink, filesDirectory)
+	if err != nil {
+		return nil, err
+	}
 	if _, err = os.Stat(filePath); err != nil {
 		if os.IsNotExist(err) {
 			return delegatorContract, nil
@@ -76,7 +83,10 @@ func Contract(network types.Network, address, protocol, filesDirectory string) (
 
 // ContractBySymLink - reads contract from file system
 func ContractBySymLink(network types.Network, address, symLink, filesDirectory string) ([]byte, error) {
-	filePath := helpers.CleanPath(fmt.Sprintf(contractFormatPath, filesDirectory, network, address, symLink))
+	filePath, err := getFilePath(network, address, symLink, filesDirectory)
+	if err != nil {
+		return nil, err
+	}
 	if _, err := os.Stat(filePath); err != nil {
 		if os.IsNotExist(err) {
 			return delegatorContract, nil
@@ -86,3 +96,21 @@ func ContractBySymLink(network types.Network, address, symLink, filesDirectory s
 	}
 	return ioutil.ReadFile(filePath)
 }
+
+func getFilePath(network types.Network, address, symLink, filesDirectory string) (string, error) {
+	if err := chechPath(filesDirectory); err != nil {
+		return "", err
+	}
+	if err := chechPath(address); err != nil {
+		return "", err
+	}
+	name := fmt.Sprintf("%s_%s.json", address, symLink)
+	return path.Join(filesDirectory, "contracts", network.String(), name), nil
+}
+
+func chechPath(path string) error {
+	if strings.Count(path, ".") > 1 {
+		return errors.Errorf("you can't change directory in share path: %s", path)
+	}
+	return nil
+}

internal/models/contract/model.go

@@ -28,9 +28,9 @@ type Contract struct {
 	Annotations          pq.StringArray `json:"annotations,omitempty" gorm:"type:text[]"`
 	Hardcoded            pq.StringArray `json:"hardcoded,omitempty" gorm:"type:text[]"`
 
-	Address  string `json:"address" gorm:"index:contracts_idx"`
-	Manager  string `json:"manager,omitempty"`
-	Delegate string `json:"delegate,omitempty"`
+	Address  string           `json:"address" gorm:"index:contracts_idx"`
+	Manager  types.NullString `json:"manager,omitempty"`
+	Delegate types.NullString `json:"delegate,omitempty"`
 
 	ProjectID       string    `json:"project_id,omitempty"`
 	TxCount         int64     `json:"tx_count" gorm:",default:0"`