From 17ba19c0cf8a80a167603ff0e1487f6b10589f37 Mon Sep 17 00:00:00 2001 From: Artem Poltorzhitskiy Date: Mon, 13 Dec 2021 16:20:20 +0300 Subject: [PATCH] Improvement: replace empty string to NULL in database (#785) * Improvement: replace empty string to NULL in database * Fix: view * Some security fixes --- cmd/api/handlers/contract.go | 2 +- cmd/api/handlers/project.go | 2 +- cmd/api/handlers/responses.go | 27 +- cmd/api/handlers/tokens.go | 6 +- cmd/indexer/scripts/head_stats_view.sql | 2 +- configs/development.yml | 34 +-- .../classification/metrics/manager_test.go | 41 ++- internal/fetch/contract.go | 48 ++- internal/models/contract/model.go | 6 +- internal/models/operation/model.go | 8 +- internal/models/transfer/model.go | 2 +- internal/models/types/null.go | 88 ++++++ internal/parsers/contract/contract.go | 15 +- internal/parsers/ledger/ledger_test.go | 21 +- .../operations/operation_group_test.go | 276 ++++++++++-------- internal/parsers/operations/transaction.go | 10 +- internal/parsers/stacktrace/stacktrace.go | 10 +- internal/parsers/transfer/events.go | 4 +- internal/parsers/transfer/transfer.go | 6 +- internal/postgres/contract/storage.go | 4 +- internal/postgres/core/histogram.go | 25 +- internal/search/contract.go | 4 +- internal/search/operation.go | 4 +- scripts/migration/main.go | 1 + scripts/migration/migrations/nullable.go | 155 ++++++++++ 25 files changed, 572 insertions(+), 229 deletions(-) create mode 100644 internal/models/types/null.go create mode 100644 scripts/migration/migrations/nullable.go diff --git a/cmd/api/handlers/contract.go b/cmd/api/handlers/contract.go index 70a4d8bf6..3e2bca4c0 100644 --- a/cmd/api/handlers/contract.go +++ b/cmd/api/handlers/contract.go @@ -85,7 +85,7 @@ func (ctx *Context) contractPostprocessing(contract contract.Contract) (Contract res.FromModel(contract) res.Alias = ctx.CachedAlias(contract.Network, contract.Address) - res.DelegateAlias = ctx.CachedAlias(contract.Network, contract.Delegate) + res.DelegateAlias = ctx.CachedAlias(contract.Network, contract.Delegate.String()) if alias, err := ctx.TZIP.Get(contract.Network, contract.Address); err == nil { res.Slug = alias.Slug diff --git a/cmd/api/handlers/project.go b/cmd/api/handlers/project.go index 3e94583f4..8f8856527 100644 --- a/cmd/api/handlers/project.go +++ b/cmd/api/handlers/project.go @@ -106,7 +106,7 @@ func (ctx *Context) GetSimilarContracts(c *gin.Context) { response.Contracts[i].FromModel(similar[i], diff) response.Contracts[i].Alias = ctx.CachedAlias(similar[i].Network, similar[i].Address) - response.Contracts[i].DelegateAlias = ctx.CachedAlias(similar[i].Network, similar[i].Delegate) + response.Contracts[i].DelegateAlias = ctx.CachedAlias(similar[i].Network, similar[i].Delegate.String()) } c.SecureJSON(http.StatusOK, response) diff --git a/cmd/api/handlers/responses.go b/cmd/api/handlers/responses.go index 369d7ffd7..a2bd6e8fa 100644 --- a/cmd/api/handlers/responses.go +++ b/cmd/api/handlers/responses.go @@ -84,7 +84,7 @@ func (o *Operation) FromModel(operation operation.Operation) { o.Delegate = operation.Delegate o.Status = operation.Status.String() o.Burned = operation.Burned - o.Entrypoint = operation.Entrypoint + o.Entrypoint = operation.Entrypoint.String() o.ContentIndex = operation.ContentIndex o.AllocatedDestinationContractBurned = operation.AllocatedDestinationContractBurned o.ConsumedGas = operation.ConsumedGas @@ -96,12 +96,11 @@ func (o *Operation) FromModel(operation operation.Operation) { // ToModel - func (o *Operation) ToModel() operation.Operation { return operation.Operation{ - ID: o.ID, - Hash: o.Hash, - Network: types.NewNetwork(o.Network), - Internal: o.Internal, - Timestamp: o.Timestamp, - + ID: o.ID, + Hash: o.Hash, + Network: types.NewNetwork(o.Network), + Internal: o.Internal, + Timestamp: o.Timestamp, Level: o.Level, Kind: types.NewOperationKind(o.Kind), Source: o.Source, @@ -114,8 +113,10 @@ func (o *Operation) ToModel() operation.Operation { Delegate: o.Delegate, Status: types.NewOperationStatus(o.Status), Burned: o.Burned, - Entrypoint: o.Entrypoint, - + Entrypoint: types.NullString{ + Str: o.Entrypoint, + Valid: o.Entrypoint != "", + }, AllocatedDestinationContract: o.AllocatedDestinationContract, ConsumedGas: o.ConsumedGas, StorageSize: o.StorageSize, @@ -161,7 +162,7 @@ type Contract struct { // FromModel - func (c *Contract) FromModel(contract contract.Contract) { c.Address = contract.Address - c.Delegate = contract.Delegate + c.Delegate = contract.Delegate.String() c.Entrypoints = contract.Entrypoints c.Hash = contract.Hash c.Language = contract.Language @@ -169,7 +170,7 @@ func (c *Contract) FromModel(contract contract.Contract) { c.LastAction = contract.LastAction c.Level = contract.Level - c.Manager = contract.Manager + c.Manager = contract.Manager.String() c.MigrationsCount = contract.MigrationsCount c.Network = contract.Network.String() c.ProjectID = contract.ProjectID @@ -470,7 +471,7 @@ func (c *SameContractsResponse) FromModel(same contract.SameResponse, ctx *Conte var contract Contract contract.FromModel(same.Contracts[i]) contract.Alias = ctx.CachedAlias(same.Contracts[i].Network, same.Contracts[i].Address) - contract.DelegateAlias = ctx.CachedAlias(same.Contracts[i].Network, same.Contracts[i].Delegate) + contract.DelegateAlias = ctx.CachedAlias(same.Contracts[i].Network, same.Contracts[i].Delegate.String()) c.Contracts[i] = contract } } @@ -535,7 +536,7 @@ func TransferFromModel(model domains.Transfer) (t Transfer) { t.To = model.To t.TokenID = model.TokenID t.Amount = model.Amount.String() - t.Parent = model.Parent + t.Parent = model.Parent.String() t.Entrypoint = model.Entrypoint t.Hash = model.Hash t.Counter = model.Counter diff --git a/cmd/api/handlers/tokens.go b/cmd/api/handlers/tokens.go index 03716ed91..c497ad0fd 100644 --- a/cmd/api/handlers/tokens.go +++ b/cmd/api/handlers/tokens.go @@ -195,10 +195,10 @@ func (ctx *Context) contractToTokens(contracts []contract.Contract, network type Level: contracts[i].Level, Timestamp: contracts[i].Timestamp, Address: contracts[i].Address, - Manager: contracts[i].Manager, - Delegate: contracts[i].Delegate, + Manager: contracts[i].Manager.String(), + Delegate: contracts[i].Delegate.String(), Alias: ctx.CachedAlias(contracts[i].Network, contracts[i].Address), - DelegateAlias: ctx.CachedAlias(contracts[i].Network, contracts[i].Delegate), + DelegateAlias: ctx.CachedAlias(contracts[i].Network, contracts[i].Delegate.String()), LastAction: contracts[i].LastAction, TxCount: contracts[i].TxCount, } diff --git a/cmd/indexer/scripts/head_stats_view.sql b/cmd/indexer/scripts/head_stats_view.sql index 2012c5d26..c9ef383a8 100644 --- a/cmd/indexer/scripts/head_stats_view.sql +++ b/cmd/indexer/scripts/head_stats_view.sql @@ -1,5 +1,5 @@ create materialized view if not exists head_stats AS -select network, count(*) as value, 'calls_count' as stats_type from operations where entrypoint != '' group by network +select network, count(*) as value, 'calls_count' as stats_type from operations where entrypoint is not null group by network union all select network, count(*) as value, 'contracts_count' as stats_type from contracts group by network union all diff --git a/configs/development.yml b/configs/development.yml index 1fd8415b7..12bfb24a1 100644 --- a/configs/development.yml +++ b/configs/development.yml @@ -2,14 +2,11 @@ rpc: mainnet: uri: https://rpc.tzkt.io/mainnet timeout: 20 - florencenet: - uri: https://rpc.tzkt.io/florencenobanet - timeout: 20 granadanet: - uri: https://granadanet.smartpy.io/ + uri: https://rpc.tzkt.io/granadanet timeout: 20 - hangzhounet: - uri: https://rpc.tzkt.io/hangzhounet + hangzhou2net: + uri: https://rpc.tzkt.io/hangzhou2net timeout: 20 tzkt: @@ -17,21 +14,21 @@ tzkt: uri: https://api.tzkt.io/v1/ base_uri: https://tzkt.io/ timeout: 20 - florencenet: - uri: https://api.florencenet.tzkt.io/v1/ - base_uri: https://florencenet.tzkt.io/ + granadanet: + uri: https://api.granadanet.tzkt.io/v1/ + base_uri: https://granadanet.tzkt.io/ timeout: 20 - hangzhounet: - uri: https://api.hangzhounet.tzkt.io/v1/ - base_uri: https://hangzhounet.tzkt.io/ + hangzhou2net: + uri: https://api.hangzhou2net.tzkt.io/v1/ + base_uri: https://hangzhou2net.tzkt.io/ timeout: 20 services: mainnet: mempool: https://mempool.dipdup.net/v1/graphql - florencenet: + granadanet: mempool: https://mempool.dipdup.net/v1/graphql - hangzhounet: + hangzhou2net: mempool: https://mempool.dipdup.net/v1/graphql storage: @@ -72,9 +69,8 @@ api: sandbox_mode: false networks: - mainnet - - florencenet - granadanet - - hangzhounet + - hangzhou2net pinata: key: ${PINATA_KEY} secret_key: ${PINATA_SECRET_KEY} @@ -90,9 +86,8 @@ indexer: networks: mainnet: boost: tzkt - florencenet: granadanet: - hangzhounet: + hangzhou2net: connections: max: 5 idle: 5 @@ -113,9 +108,8 @@ scripts: secret_access_key: ${AWS_SECRET_ACCESS_KEY} networks: - mainnet - - florencenet - granadanet - - hangzhounet + - hangzhou2net connections: max: 5 idle: 5 diff --git a/internal/classification/metrics/manager_test.go b/internal/classification/metrics/manager_test.go index 2b18f3620..8e141992a 100644 --- a/internal/classification/metrics/manager_test.go +++ b/internal/classification/metrics/manager_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/baking-bad/bcdhub/internal/models/contract" + "github.com/baking-bad/bcdhub/internal/models/types" ) func TestManager_Compute(t *testing.T) { @@ -21,11 +22,17 @@ func TestManager_Compute(t *testing.T) { name: "Case 1", args: args{ a: contract.Contract{ - Manager: "test", + Manager: types.NullString{ + Str: "test", + Valid: true, + }, Network: 0, }, b: contract.Contract{ - Manager: "test", + Manager: types.NullString{ + Str: "test", + Valid: true, + }, Network: 0, }, }, @@ -37,11 +44,17 @@ func TestManager_Compute(t *testing.T) { name: "Case 2", args: args{ a: contract.Contract{ - Manager: "other", + Manager: types.NullString{ + Str: "other", + Valid: true, + }, Network: 1, }, b: contract.Contract{ - Manager: "test", + Manager: types.NullString{ + Str: "test", + Valid: true, + }, Network: 1, }, }, @@ -53,11 +66,17 @@ func TestManager_Compute(t *testing.T) { name: "Case 3", args: args{ a: contract.Contract{ - Manager: "test", + Manager: types.NullString{ + Str: "test", + Valid: true, + }, Network: 1, }, b: contract.Contract{ - Manager: "test", + Manager: types.NullString{ + Str: "test", + Valid: true, + }, Network: 2, }, }, @@ -69,11 +88,17 @@ func TestManager_Compute(t *testing.T) { name: "Case 4", args: args{ a: contract.Contract{ - Manager: "other", + Manager: types.NullString{ + Str: "other", + Valid: true, + }, Network: 1, }, b: contract.Contract{ - Manager: "test", + Manager: types.NullString{ + Str: "test", + Valid: true, + }, Network: 2, }, }, diff --git a/internal/fetch/contract.go b/internal/fetch/contract.go index 65b384330..8527c42f0 100644 --- a/internal/fetch/contract.go +++ b/internal/fetch/contract.go @@ -4,17 +4,14 @@ import ( "fmt" "io/ioutil" "os" + "path" + "strings" "github.com/baking-bad/bcdhub/internal/bcd" - "github.com/baking-bad/bcdhub/internal/helpers" "github.com/baking-bad/bcdhub/internal/models/types" "github.com/pkg/errors" ) -const ( - contractFormatPath = "%s/contracts/%s/%s_%s.json" -) - var ( delegatorContract = []byte(`[{"prim":"parameter","args":[{"prim":"or","args":[{"prim":"lambda","args":[{"prim":"unit"},{"prim":"list","args":[{"prim":"operation"}]}],"annots":["%do"]},{"prim":"unit","annots":["%default"]}]}]},{"prim":"storage","args":[{"prim":"key_hash"}]},{"prim":"code","args":[[[[{"prim":"DUP"},{"prim":"CAR"},{"prim":"DIP","args":[[{"prim":"CDR"}]]}]],{"prim":"IF_LEFT","args":[[{"prim":"PUSH","args":[{"prim":"mutez"},{"int":"0"}]},{"prim":"AMOUNT"},[[{"prim":"COMPARE"},{"prim":"EQ"}],{"prim":"IF","args":[[],[[{"prim":"UNIT"},{"prim":"FAILWITH"}]]]}],[{"prim":"DIP","args":[[{"prim":"DUP"}]]},{"prim":"SWAP"}],{"prim":"IMPLICIT_ACCOUNT"},{"prim":"ADDRESS"},{"prim":"SENDER"},[[{"prim":"COMPARE"},{"prim":"EQ"}],{"prim":"IF","args":[[],[[{"prim":"UNIT"},{"prim":"FAILWITH"}]]]}],{"prim":"UNIT"},{"prim":"EXEC"},{"prim":"PAIR"}],[{"prim":"DROP"},{"prim":"NIL","args":[{"prim":"operation"}]},{"prim":"PAIR"}]]}]]}]`) ) @@ -29,7 +26,10 @@ func RemoveContract(network types.Network, address, protocol, filesDirectory str return err } - filePath := fmt.Sprintf(contractFormatPath, filesDirectory, network, address, protoSymLink) + filePath, err := getFilePath(network, address, protoSymLink, filesDirectory) + if err != nil { + return err + } if _, err = os.Stat(filePath); err == nil { return os.Remove(filePath) } else if !os.IsNotExist(err) { @@ -39,12 +39,16 @@ func RemoveContract(network types.Network, address, protocol, filesDirectory str } // RemoveAllContracts - -func RemoveAllContracts(network types.Network, filesDirectory string) error { +func RemoveAllContracts(network fmt.Stringer, filesDirectory string) error { if filesDirectory == "" { return errors.Errorf("Invalid filesDirectory: %s", filesDirectory) } - dirPath := fmt.Sprintf("%s/contracts/%s", filesDirectory, network) + if err := chechPath(filesDirectory); err != nil { + return err + } + + dirPath := path.Join(filesDirectory, "contracts", network.String()) if _, err := os.Stat(dirPath); err == nil { return os.RemoveAll(dirPath) } else if !os.IsNotExist(err) { @@ -63,7 +67,10 @@ func Contract(network types.Network, address, protocol, filesDirectory string) ( return nil, err } - filePath := helpers.CleanPath(fmt.Sprintf(contractFormatPath, filesDirectory, network, address, protoSymLink)) + filePath, err := getFilePath(network, address, protoSymLink, filesDirectory) + if err != nil { + return nil, err + } if _, err = os.Stat(filePath); err != nil { if os.IsNotExist(err) { return delegatorContract, nil @@ -76,7 +83,10 @@ func Contract(network types.Network, address, protocol, filesDirectory string) ( // ContractBySymLink - reads contract from file system func ContractBySymLink(network types.Network, address, symLink, filesDirectory string) ([]byte, error) { - filePath := helpers.CleanPath(fmt.Sprintf(contractFormatPath, filesDirectory, network, address, symLink)) + filePath, err := getFilePath(network, address, symLink, filesDirectory) + if err != nil { + return nil, err + } if _, err := os.Stat(filePath); err != nil { if os.IsNotExist(err) { return delegatorContract, nil @@ -86,3 +96,21 @@ func ContractBySymLink(network types.Network, address, symLink, filesDirectory s } return ioutil.ReadFile(filePath) } + +func getFilePath(network types.Network, address, symLink, filesDirectory string) (string, error) { + if err := chechPath(filesDirectory); err != nil { + return "", err + } + if err := chechPath(address); err != nil { + return "", err + } + name := fmt.Sprintf("%s_%s.json", address, symLink) + return path.Join(filesDirectory, "contracts", network.String(), name), nil +} + +func chechPath(path string) error { + if strings.Count(path, ".") > 1 { + return errors.Errorf("you can't change directory in share path: %s", path) + } + return nil +} diff --git a/internal/models/contract/model.go b/internal/models/contract/model.go index 7daddf672..2778310c5 100644 --- a/internal/models/contract/model.go +++ b/internal/models/contract/model.go @@ -28,9 +28,9 @@ type Contract struct { Annotations pq.StringArray `json:"annotations,omitempty" gorm:"type:text[]"` Hardcoded pq.StringArray `json:"hardcoded,omitempty" gorm:"type:text[]"` - Address string `json:"address" gorm:"index:contracts_idx"` - Manager string `json:"manager,omitempty"` - Delegate string `json:"delegate,omitempty"` + Address string `json:"address" gorm:"index:contracts_idx"` + Manager types.NullString `json:"manager,omitempty"` + Delegate types.NullString `json:"delegate,omitempty"` ProjectID string `json:"project_id,omitempty"` TxCount int64 `json:"tx_count" gorm:",default:0"` diff --git a/internal/models/operation/model.go b/internal/models/operation/model.go index 527f59e0a..9aea11387 100644 --- a/internal/models/operation/model.go +++ b/internal/models/operation/model.go @@ -35,7 +35,7 @@ type Operation struct { AllocatedDestinationContractBurned int64 `json:"allocated_destination_contract_burned,omitempty"` Nonce *int64 `json:"nonce,omitempty"` - Network types.Network `json:"network" gorm:"type:SMALLINT;index:idx_operations_level_network"` + Network types.Network `json:"network" gorm:"type:SMALLINT;index:idx_operations_level_network; index:operations_network_idx"` ProtocolID int64 `json:"protocol" gorm:"type:SMALLINT"` Hash string `json:"hash" gorm:"index:new_opg_idx;index:operations_hash_idx"` @@ -46,7 +46,7 @@ type Operation struct { Source string `json:"source" gorm:"index:source_idx"` Destination string `json:"destination,omitempty" gorm:"index:destination_idx"` Delegate string `json:"delegate,omitempty"` - Entrypoint string `json:"entrypoint,omitempty"` + Entrypoint types.NullString `json:"entrypoint,omitempty" gorm:"index:operations_entrypoint_idx"` Parameters []byte `json:"parameters,omitempty"` DeffatedStorage []byte `json:"deffated_storage"` @@ -119,7 +119,7 @@ func (o *Operation) SetBurned(constants protocol.Constants) { // IsEntrypoint - func (o *Operation) IsEntrypoint(entrypoint string) bool { - return o.Entrypoint == entrypoint + return o.Entrypoint.EqualString(entrypoint) } // IsOrigination - @@ -151,7 +151,7 @@ func (o Operation) EmptyTransfer() *transfer.Transfer { Timestamp: o.Timestamp, Level: o.Level, Initiator: o.Source, - Entrypoint: o.Entrypoint, + Entrypoint: o.Entrypoint.String(), Amount: decimal.Zero, OperationID: o.ID, } diff --git a/internal/models/transfer/model.go b/internal/models/transfer/model.go index 14c6448af..d279bc266 100644 --- a/internal/models/transfer/model.go +++ b/internal/models/transfer/model.go @@ -24,7 +24,7 @@ type Transfer struct { To string `json:"to" gorm:"index:transfers_to_idx"` TokenID uint64 `json:"token_id" gorm:"type:numeric(50,0);index:transfers_token_idx"` Amount decimal.Decimal `json:"amount" gorm:"type:numeric(100,0)"` - Parent string `json:"parent,omitempty"` + Parent types.NullString `json:"parent,omitempty"` Entrypoint string `json:"entrypoint,omitempty"` OperationID int64 `json:"-" gorm:"index:transfers_operation_id_idx"` } diff --git a/internal/models/types/null.go b/internal/models/types/null.go new file mode 100644 index 000000000..23ef53e1a --- /dev/null +++ b/internal/models/types/null.go @@ -0,0 +1,88 @@ +package types + +import "database/sql/driver" + +// NullString represents a string that may be null. +// NullString implements the Scanner interface so +// it can be used as a scan destination: +// +// var s NullString +// err := db.QueryRow("SELECT name FROM foo WHERE id=?", id).Scan(&s) +// ... +// if s.Valid { +// // use s.Str +// } else { +// // NULL value +// } +// +type NullString struct { + Str string + Valid bool // Valid is true if Str is not NULL +} + +// NewNullString - +func NewNullString(val *string) NullString { + if val == nil { + return NullString{"", false} + } + + return NullString{*val, true} +} + +// UnmarshalJSON - +func (ns *NullString) UnmarshalJSON(data []byte) error { + if data == nil || len(data) < 2 { + ns.Valid = false + return nil + } + + ns.Valid = true + ns.Str = string(data)[0 : len(data)-1] + return nil +} + +// MarshalJSON - +func (ns NullString) MarshalJSON() ([]byte, error) { + if ns.Valid { + return []byte(ns.Str), nil + } + return nil, nil +} + +// Scan implements the Scanner interface. +func (ns *NullString) Scan(value interface{}) error { + ns.Str, ns.Valid = "", false + + if value == nil { + return nil + } + + if val, ok := value.(string); ok { + ns.Str, ns.Valid = val, true + } + return nil +} + +// Value implements the driver Valuer interface. +func (ns NullString) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return ns.Str, nil +} + +// String - +func (ns NullString) String() string { + if !ns.Valid { + return "" + } + return ns.Str +} + +// EqualString - +func (ns NullString) EqualString(value string) bool { + if !ns.Valid { + return false + } + return value == ns.Str +} diff --git a/internal/parsers/contract/contract.go b/internal/parsers/contract/contract.go index 0ed6d74a1..bc044e52f 100644 --- a/internal/parsers/contract/contract.go +++ b/internal/parsers/contract/contract.go @@ -49,12 +49,15 @@ func (p *Parser) Parse(operation *operation.Operation) (*parsers.Result, error) } contract := contract.Contract{ - Network: operation.Network, - Level: operation.Level, - Timestamp: operation.Timestamp, - Manager: operation.Source, - Address: operation.Destination, - Delegate: operation.Delegate, + Network: operation.Network, + Level: operation.Level, + Timestamp: operation.Timestamp, + Manager: types.NewNullString(&operation.Source), + Address: operation.Destination, + Delegate: types.NullString{ + Str: operation.Delegate, + Valid: operation.Delegate != "", + }, LastAction: operation.Timestamp, } diff --git a/internal/parsers/ledger/ledger_test.go b/internal/parsers/ledger/ledger_test.go index e407a7256..78d5a4623 100644 --- a/internal/parsers/ledger/ledger_test.go +++ b/internal/parsers/ledger/ledger_test.go @@ -51,8 +51,11 @@ func TestLedger_Parse(t *testing.T) { { name: "test 1", operation: &operation.Operation{ - Tags: types.FA12Tag, - Entrypoint: consts.TransferEntrypoint, + Tags: types.FA12Tag, + Entrypoint: types.NullString{ + Str: consts.TransferEntrypoint, + Valid: true, + }, Network: types.Mainnet, DeffatedStorage: []byte(`{"int":257}`), BigMapDiffs: []*bigmapdiff.BigMapDiff{ @@ -75,8 +78,11 @@ func TestLedger_Parse(t *testing.T) { }, { name: "test 2", operation: &operation.Operation{ - Tags: types.FA12Tag | types.LedgerTag, - Entrypoint: consts.TransferEntrypoint, + Tags: types.FA12Tag | types.LedgerTag, + Entrypoint: types.NullString{ + Str: consts.TransferEntrypoint, + Valid: true, + }, Network: types.Mainnet, DeffatedStorage: []byte(`{"int":257}`), BigMapDiffs: []*bigmapdiff.BigMapDiff{ @@ -99,8 +105,11 @@ func TestLedger_Parse(t *testing.T) { }, { name: "test 3", operation: &operation.Operation{ - Tags: types.FA2Tag | types.LedgerTag, - Entrypoint: "burn", + Tags: types.FA2Tag | types.LedgerTag, + Entrypoint: types.NullString{ + Str: "burn", + Valid: true, + }, Kind: types.OperationKindTransaction, Network: types.Mainnet, Destination: "KT1981tPmXh4KrUQKZpQKb55kREX7QGJcF3E", diff --git a/internal/parsers/operations/operation_group_test.go b/internal/parsers/operations/operation_group_test.go index a4efc1798..9365fe9ac 100644 --- a/internal/parsers/operations/operation_group_test.go +++ b/internal/parsers/operations/operation_group_test.go @@ -386,18 +386,21 @@ func TestGroup_Parse(t *testing.T) { want: &parsers.Result{ Operations: []*operation.Operation{ { - Kind: types.OperationKindTransaction, - Source: "tz1aSPEN4RTZbn4aXEsxDiix38dDmacGQ8sq", - Fee: 37300, - Counter: 5791164, - GasLimit: 369423, - StorageLimit: 90, - Destination: "KT1S5iPRQ612wcNm6mXDqDhTNegGFcvTV7vM", - Status: types.OperationStatusApplied, - Level: 1068669, - Network: types.Mainnet, - Hash: "opJXaAMkBrAbd1XFd23kS8vXiw63tU4rLUcLrZgqUCpCbhT1Pn9", - Entrypoint: "transfer", + Kind: types.OperationKindTransaction, + Source: "tz1aSPEN4RTZbn4aXEsxDiix38dDmacGQ8sq", + Fee: 37300, + Counter: 5791164, + GasLimit: 369423, + StorageLimit: 90, + Destination: "KT1S5iPRQ612wcNm6mXDqDhTNegGFcvTV7vM", + Status: types.OperationStatusApplied, + Level: 1068669, + Network: types.Mainnet, + Hash: "opJXaAMkBrAbd1XFd23kS8vXiw63tU4rLUcLrZgqUCpCbhT1Pn9", + Entrypoint: types.NullString{ + Str: "transfer", + Valid: true, + }, Timestamp: timestamp, Burned: 70000, Initiator: "tz1aSPEN4RTZbn4aXEsxDiix38dDmacGQ8sq", @@ -445,16 +448,19 @@ func TestGroup_Parse(t *testing.T) { }, }, }, { - Kind: types.OperationKindTransaction, - Source: "KT1S5iPRQ612wcNm6mXDqDhTNegGFcvTV7vM", - Destination: "KT19nHqEWZxFFbbDL1b7Y86escgEN7qUShGo", - Status: types.OperationStatusApplied, - Level: 1068669, - Counter: 5791164, - Network: types.Mainnet, - Hash: "opJXaAMkBrAbd1XFd23kS8vXiw63tU4rLUcLrZgqUCpCbhT1Pn9", - Nonce: setInt64(0), - Entrypoint: "validateAccounts", + Kind: types.OperationKindTransaction, + Source: "KT1S5iPRQ612wcNm6mXDqDhTNegGFcvTV7vM", + Destination: "KT19nHqEWZxFFbbDL1b7Y86escgEN7qUShGo", + Status: types.OperationStatusApplied, + Level: 1068669, + Counter: 5791164, + Network: types.Mainnet, + Hash: "opJXaAMkBrAbd1XFd23kS8vXiw63tU4rLUcLrZgqUCpCbhT1Pn9", + Nonce: setInt64(0), + Entrypoint: types.NullString{ + Str: "validateAccounts", + Valid: true, + }, Internal: true, Timestamp: timestamp, ProtocolID: 1, @@ -462,16 +468,19 @@ func TestGroup_Parse(t *testing.T) { Parameters: []byte("{\"entrypoint\":\"validateAccounts\",\"value\":{\"prim\":\"Pair\",\"args\":[{\"prim\":\"Pair\",\"args\":[{\"prim\":\"Pair\",\"args\":[{\"prim\":\"Pair\",\"args\":[{\"bytes\":\"0000a2560a416161def96031630886abe950c4baf036\"},{\"bytes\":\"0000fdf98b65d53a9661e07f41093dcb6f3d931736ba\"}]},{\"prim\":\"Pair\",\"args\":[{\"int\":\"14151000\"},{\"int\":\"0\"}]}]},{\"prim\":\"Pair\",\"args\":[{\"prim\":\"True\"},{\"prim\":\"Pair\",\"args\":[{\"int\":\"8010000\"},{\"int\":\"18000000\"}]}]}]},{\"bytes\":\"01796ad78734892d5ae4186e84a30290040732ada70076616c696461746552756c6573\"}]}}"), DeffatedStorage: []byte("{\"int\":\"61\"}"), }, { - Kind: types.OperationKindTransaction, - Source: "KT19nHqEWZxFFbbDL1b7Y86escgEN7qUShGo", - Destination: "KT1KemKUx79keZgFW756jQrqKcZJ21y4SPdS", - Status: types.OperationStatusApplied, - Level: 1068669, - Counter: 5791164, - Network: types.Mainnet, - Hash: "opJXaAMkBrAbd1XFd23kS8vXiw63tU4rLUcLrZgqUCpCbhT1Pn9", - Nonce: setInt64(1), - Entrypoint: "validateRules", + Kind: types.OperationKindTransaction, + Source: "KT19nHqEWZxFFbbDL1b7Y86escgEN7qUShGo", + Destination: "KT1KemKUx79keZgFW756jQrqKcZJ21y4SPdS", + Status: types.OperationStatusApplied, + Level: 1068669, + Counter: 5791164, + Network: types.Mainnet, + Hash: "opJXaAMkBrAbd1XFd23kS8vXiw63tU4rLUcLrZgqUCpCbhT1Pn9", + Nonce: setInt64(1), + Entrypoint: types.NullString{ + Str: "validateRules", + Valid: true, + }, Internal: true, Timestamp: timestamp, ProtocolID: 1, @@ -554,25 +563,28 @@ func TestGroup_Parse(t *testing.T) { want: &parsers.Result{ Operations: []*operation.Operation{ { - ContentIndex: 0, - Network: types.Mainnet, - ProtocolID: 1, - Hash: "opPUPCpQu6pP38z9TkgFfwLiqVBFGSWQCH8Z2PUL3jrpxqJH5gt", - Internal: false, - Nonce: nil, - Status: types.OperationStatusApplied, - Timestamp: timestamp, - Level: 1151495, - Kind: types.OperationKindTransaction, - Initiator: "tz1dMH7tW7RhdvVMR4wKVFF1Ke8m8ZDvrTTE", - Source: "tz1dMH7tW7RhdvVMR4wKVFF1Ke8m8ZDvrTTE", - Fee: 43074, - Counter: 6909186, - GasLimit: 427673, - StorageLimit: 47, - Destination: "KT1Ap287P1NzsnToSJdA4aqSNjPomRaHBZSr", - Parameters: []byte("{\"entrypoint\":\"redeem\",\"value\":{\"bytes\":\"a874aac22777351417c9bde0920cc7ed33e54453e1dd149a1f3a60521358d19a\"}}"), - Entrypoint: "redeem", + ContentIndex: 0, + Network: types.Mainnet, + ProtocolID: 1, + Hash: "opPUPCpQu6pP38z9TkgFfwLiqVBFGSWQCH8Z2PUL3jrpxqJH5gt", + Internal: false, + Nonce: nil, + Status: types.OperationStatusApplied, + Timestamp: timestamp, + Level: 1151495, + Kind: types.OperationKindTransaction, + Initiator: "tz1dMH7tW7RhdvVMR4wKVFF1Ke8m8ZDvrTTE", + Source: "tz1dMH7tW7RhdvVMR4wKVFF1Ke8m8ZDvrTTE", + Fee: 43074, + Counter: 6909186, + GasLimit: 427673, + StorageLimit: 47, + Destination: "KT1Ap287P1NzsnToSJdA4aqSNjPomRaHBZSr", + Parameters: []byte("{\"entrypoint\":\"redeem\",\"value\":{\"bytes\":\"a874aac22777351417c9bde0920cc7ed33e54453e1dd149a1f3a60521358d19a\"}}"), + Entrypoint: types.NullString{ + Str: "redeem", + Valid: true, + }, DeffatedStorage: []byte("{\"prim\":\"Pair\",\"args\":[{\"int\":\"32\"},{\"prim\":\"Unit\"}]}"), BigMapDiffs: []*bigmapdiff.BigMapDiff{ { @@ -587,22 +599,25 @@ func TestGroup_Parse(t *testing.T) { }, }, }, { - ContentIndex: 0, - Network: types.Mainnet, - ProtocolID: 1, - Hash: "opPUPCpQu6pP38z9TkgFfwLiqVBFGSWQCH8Z2PUL3jrpxqJH5gt", - Internal: true, - Nonce: setInt64(0), - Status: types.OperationStatusApplied, - Timestamp: timestamp, - Level: 1151495, - Kind: types.OperationKindTransaction, - Initiator: "tz1dMH7tW7RhdvVMR4wKVFF1Ke8m8ZDvrTTE", - Source: "KT1Ap287P1NzsnToSJdA4aqSNjPomRaHBZSr", - Counter: 6909186, - Destination: "KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn", - Parameters: []byte("{\"entrypoint\":\"transfer\",\"value\":{\"prim\":\"Pair\",\"args\":[{\"bytes\":\"011871cfab6dafee00330602b4342b6500c874c93b00\"},{\"prim\":\"Pair\",\"args\":[{\"bytes\":\"0000c2473c617946ce7b9f6843f193401203851cb2ec\"},{\"int\":\"7874880\"}]}]}}"), - Entrypoint: "transfer", + ContentIndex: 0, + Network: types.Mainnet, + ProtocolID: 1, + Hash: "opPUPCpQu6pP38z9TkgFfwLiqVBFGSWQCH8Z2PUL3jrpxqJH5gt", + Internal: true, + Nonce: setInt64(0), + Status: types.OperationStatusApplied, + Timestamp: timestamp, + Level: 1151495, + Kind: types.OperationKindTransaction, + Initiator: "tz1dMH7tW7RhdvVMR4wKVFF1Ke8m8ZDvrTTE", + Source: "KT1Ap287P1NzsnToSJdA4aqSNjPomRaHBZSr", + Counter: 6909186, + Destination: "KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn", + Parameters: []byte("{\"entrypoint\":\"transfer\",\"value\":{\"prim\":\"Pair\",\"args\":[{\"bytes\":\"011871cfab6dafee00330602b4342b6500c874c93b00\"},{\"prim\":\"Pair\",\"args\":[{\"bytes\":\"0000c2473c617946ce7b9f6843f193401203851cb2ec\"},{\"int\":\"7874880\"}]}]}}"), + Entrypoint: types.NullString{ + Str: "transfer", + Valid: true, + }, Burned: 47000, DeffatedStorage: []byte("{\"prim\":\"Pair\",\"args\":[{\"int\":\"31\"},{\"prim\":\"Pair\",\"args\":[[{\"prim\":\"DUP\"},{\"prim\":\"CAR\"},{\"prim\":\"DIP\",\"args\":[[{\"prim\":\"CDR\"}]]},{\"prim\":\"DUP\"},{\"prim\":\"DUP\"},{\"prim\":\"CAR\"},{\"prim\":\"DIP\",\"args\":[[{\"prim\":\"CDR\"}]]},{\"prim\":\"DIP\",\"args\":[[{\"prim\":\"DIP\",\"args\":[{\"int\":\"2\"},[{\"prim\":\"DUP\"}]]},{\"prim\":\"DIG\",\"args\":[{\"int\":\"2\"}]}]]},{\"prim\":\"PUSH\",\"args\":[{\"prim\":\"string\"},{\"string\":\"code\"}]},{\"prim\":\"PAIR\"},{\"prim\":\"PACK\"},{\"prim\":\"GET\"},{\"prim\":\"IF_NONE\",\"args\":[[{\"prim\":\"NONE\",\"args\":[{\"prim\":\"lambda\",\"args\":[{\"prim\":\"pair\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"big_map\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"bytes\"}]}]},{\"prim\":\"pair\",\"args\":[{\"prim\":\"list\",\"args\":[{\"prim\":\"operation\"}]},{\"prim\":\"big_map\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"bytes\"}]}]}]}]}],[{\"prim\":\"UNPACK\",\"args\":[{\"prim\":\"lambda\",\"args\":[{\"prim\":\"pair\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"big_map\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"bytes\"}]}]},{\"prim\":\"pair\",\"args\":[{\"prim\":\"list\",\"args\":[{\"prim\":\"operation\"}]},{\"prim\":\"big_map\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"bytes\"}]}]}]}]},{\"prim\":\"IF_NONE\",\"args\":[[{\"prim\":\"PUSH\",\"args\":[{\"prim\":\"string\"},{\"string\":\"UStore: failed to unpack code\"}]},{\"prim\":\"FAILWITH\"}],[]]},{\"prim\":\"SOME\"}]]},{\"prim\":\"IF_NONE\",\"args\":[[{\"prim\":\"DROP\"},{\"prim\":\"DIP\",\"args\":[[{\"prim\":\"DUP\"},{\"prim\":\"PUSH\",\"args\":[{\"prim\":\"bytes\"},{\"bytes\":\"05010000000866616c6c6261636b\"}]},{\"prim\":\"GET\"},{\"prim\":\"IF_NONE\",\"args\":[[{\"prim\":\"PUSH\",\"args\":[{\"prim\":\"string\"},{\"string\":\"UStore: no field fallback\"}]},{\"prim\":\"FAILWITH\"}],[]]},{\"prim\":\"UNPACK\",\"args\":[{\"prim\":\"lambda\",\"args\":[{\"prim\":\"pair\",\"args\":[{\"prim\":\"pair\",\"args\":[{\"prim\":\"string\"},{\"prim\":\"bytes\"}]},{\"prim\":\"big_map\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"bytes\"}]}]},{\"prim\":\"pair\",\"args\":[{\"prim\":\"list\",\"args\":[{\"prim\":\"operation\"}]},{\"prim\":\"big_map\",\"args\":[{\"prim\":\"bytes\"},{\"prim\":\"bytes\"}]}]}]}]},{\"prim\":\"IF_NONE\",\"args\":[[{\"prim\":\"PUSH\",\"args\":[{\"prim\":\"string\"},{\"string\":\"UStore: failed to unpack fallback\"}]},{\"prim\":\"FAILWITH\"}],[]]},{\"prim\":\"SWAP\"}]]},{\"prim\":\"PAIR\"},{\"prim\":\"EXEC\"}],[{\"prim\":\"DIP\",\"args\":[[{\"prim\":\"SWAP\"},{\"prim\":\"DROP\"},{\"prim\":\"PAIR\"}]]},{\"prim\":\"SWAP\"},{\"prim\":\"EXEC\"}]]}],{\"prim\":\"Pair\",\"args\":[{\"int\":\"1\"},{\"prim\":\"False\"}]}]}]}"), Tags: types.FA12Tag, @@ -780,7 +795,10 @@ func TestGroup_Parse(t *testing.T) { Hash: "97a40c7ff3bad5edb92c8e1dcfd4bfc778da8166a7632c1bcecbf8d8f9e4490b", Entrypoints: []string{"decrement", "increment"}, Address: "KT1NppzrgyLZD3aku7fssfhYPm5QqZwyabvR", - Manager: "tz1SX7SPdx4ZJb6uP5Hh5XBVZhh9wTfFaud3", + Manager: types.NullString{ + Str: "tz1SX7SPdx4ZJb6uP5Hh5XBVZhh9wTfFaud3", + Valid: true, + }, }, }, }, @@ -849,7 +867,10 @@ func TestGroup_Parse(t *testing.T) { Tags: types.Tags(0), Entrypoints: []string{"default"}, Address: "KT1AbjG7vtpV8osdoJXcMRck8eTwst8dWoz4", - Manager: "tz1MXrEgDNnR8PDryN8sq4B2m9Pqcf57wBqM", + Manager: types.NullString{ + Str: "tz1MXrEgDNnR8PDryN8sq4B2m9Pqcf57wBqM", + Valid: true, + }, }, }, }, @@ -890,19 +911,22 @@ func TestGroup_Parse(t *testing.T) { want: &parsers.Result{ Operations: []*operation.Operation{ { - Kind: types.OperationKindTransaction, - Source: "tz1gXhGAXgKvrXjn4t16rYUXocqbch1XXJFN", - Fee: 4045, - Counter: 155670, - GasLimit: 37831, - StorageLimit: 5265, - Destination: "KT1C2MfcjWb5R1ZDDxVULCsGuxrf5fEn5264", - Status: types.OperationStatusApplied, - Level: 72207, - Network: types.Edo2net, - Hash: "op4fFMvYsxvSUKZmLWC7aUf25VMYqigaDwTZCAoBBi8zACbHTNg", - Timestamp: timestamp, - Entrypoint: "@entrypoint_1", + Kind: types.OperationKindTransaction, + Source: "tz1gXhGAXgKvrXjn4t16rYUXocqbch1XXJFN", + Fee: 4045, + Counter: 155670, + GasLimit: 37831, + StorageLimit: 5265, + Destination: "KT1C2MfcjWb5R1ZDDxVULCsGuxrf5fEn5264", + Status: types.OperationStatusApplied, + Level: 72207, + Network: types.Edo2net, + Hash: "op4fFMvYsxvSUKZmLWC7aUf25VMYqigaDwTZCAoBBi8zACbHTNg", + Timestamp: timestamp, + Entrypoint: types.NullString{ + Str: "@entrypoint_1", + Valid: true, + }, Initiator: "tz1gXhGAXgKvrXjn4t16rYUXocqbch1XXJFN", Parameters: []byte("{\"entrypoint\":\"default\",\"value\":{\"prim\":\"Right\",\"args\":[{\"prim\":\"Unit\"}]}}"), ProtocolID: 3, @@ -1002,7 +1026,10 @@ func TestGroup_Parse(t *testing.T) { Annotations: []string{"%token_address", "%drop_proposal", "%transfer_contract_tokens", "%permits_counter", "%remove_operator", "%mint", "%ledger", "%voters", "%owner", "%balance", "%transfer", "%from_", "%max_voting_period", "%not_in_migration", "%start_date", "%custom_entrypoints", "%proposal_check", "%accept_ownership", "%migrate", "%set_quorum_threshold", "%amount", "%proposals", "%min_voting_period", "%rejected_proposal_return_value", "%burn", "%flush", "%max_quorum_threshold", "%migratingTo", "%operators", "%proposer", "%call_FA2", "%argument", "%params", "%transfer_ownership", "%voting_period", "%request", "%confirm_migration", "%frozen_token", "%param", "%admin", "%migration_status", "%proposal_key_list_sort_by_date", "%requests", "%update_operators", "%add_operator", "%getVotePermitCounter", "%propose", "%vote", "%vote_amount", "%proposer_frozen_token", "%callCustom", "%txs", "%operator", "%quorum_threshold", "%to_", "%set_voting_period", "%callback", "%contract_address", "%downvotes", "%max_votes", "%balance_of", "%proposal_key", "%vote_type", "%signature", "%decision_lambda", "%token_id", "%permit", "%key", "%extra", "%pending_owner", "%upvotes", "%max_proposals", "%min_quorum_threshold", "%proposal_metadata", "%metadata", "%migratedTo"}, Entrypoints: []string{"callCustom", "accept_ownership", "burn", "balance_of", "transfer", "update_operators", "confirm_migration", "drop_proposal", "flush", "getVotePermitCounter", "migrate", "mint", "propose", "set_quorum_threshold", "set_voting_period", "transfer_ownership", "vote", "transfer_contract_tokens"}, Address: "KT1JgHoXtZPjVfG82BY3FSys2VJhKVZo2EJU", - Manager: "KT1C2MfcjWb5R1ZDDxVULCsGuxrf5fEn5264", + Manager: types.NullString{ + Str: "KT1C2MfcjWb5R1ZDDxVULCsGuxrf5fEn5264", + Valid: true, + }, }, }, }, @@ -1042,18 +1069,21 @@ func TestGroup_Parse(t *testing.T) { want: &parsers.Result{ Operations: []*operation.Operation{ { - Kind: types.OperationKindTransaction, - Source: "tz1aCzsYRUgDZBV7zb7Si6q2AobrocFW5qwb", - Fee: 2235, - Counter: 9432992, - GasLimit: 18553, - Destination: "KT1QcxwB4QyPKfmSwjH1VRxa6kquUjeDWeEy", - Status: types.OperationStatusApplied, - Level: 1516349, - Network: types.Mainnet, - Hash: "ooz1bkCQeYsZYP7vb4Dx7pYPRpWN11Z3G3yP1v4HAfdNXuHRv9c", - Timestamp: timestamp, - Entrypoint: "transfer", + Kind: types.OperationKindTransaction, + Source: "tz1aCzsYRUgDZBV7zb7Si6q2AobrocFW5qwb", + Fee: 2235, + Counter: 9432992, + GasLimit: 18553, + Destination: "KT1QcxwB4QyPKfmSwjH1VRxa6kquUjeDWeEy", + Status: types.OperationStatusApplied, + Level: 1516349, + Network: types.Mainnet, + Hash: "ooz1bkCQeYsZYP7vb4Dx7pYPRpWN11Z3G3yP1v4HAfdNXuHRv9c", + Timestamp: timestamp, + Entrypoint: types.NullString{ + Str: "transfer", + Valid: true, + }, Tags: types.FA2Tag | types.LedgerTag, Initiator: "tz1aCzsYRUgDZBV7zb7Si6q2AobrocFW5qwb", Parameters: []byte(`{"entrypoint":"transfer","value":[{"prim":"Pair","args":[{"string":"tz1aCzsYRUgDZBV7zb7Si6q2AobrocFW5qwb"},[{"prim":"Pair","args":[{"string":"tz1a6ZKyEoCmfpsY74jEq6uKBK8RQXdj1aVi"},{"prim":"Pair","args":[{"int":"12"},{"int":"1"}]}]}]]}]}`), @@ -1160,37 +1190,43 @@ func TestGroup_Parse(t *testing.T) { }, Operations: []*operation.Operation{ { - Kind: types.OperationKindTransaction, - Hash: "oocFt4vkkgQGfoRH54328cJUbDdWvj3x6KEs5Arm4XhqwwJmnJ8", - Source: "tz1WKygtstVY96oyc6Rmk945dMf33LeihgWT", - Initiator: "tz1WKygtstVY96oyc6Rmk945dMf33LeihgWT", - Status: types.OperationStatusApplied, - Fee: 5043, - Counter: 10671622, - GasLimit: 46511, - StorageLimit: 400, - Amount: 0, - Timestamp: timestamp, - Level: 1520888, - Network: types.Mainnet, - Entrypoint: "update_record", + Kind: types.OperationKindTransaction, + Hash: "oocFt4vkkgQGfoRH54328cJUbDdWvj3x6KEs5Arm4XhqwwJmnJ8", + Source: "tz1WKygtstVY96oyc6Rmk945dMf33LeihgWT", + Initiator: "tz1WKygtstVY96oyc6Rmk945dMf33LeihgWT", + Status: types.OperationStatusApplied, + Fee: 5043, + Counter: 10671622, + GasLimit: 46511, + StorageLimit: 400, + Amount: 0, + Timestamp: timestamp, + Level: 1520888, + Network: types.Mainnet, + Entrypoint: types.NullString{ + Str: "update_record", + Valid: true, + }, ProtocolID: 4, Destination: "KT1H1MqmUM4aK9i1833EBmYCCEfkbt6ZdSBc", Parameters: []byte(`{"entrypoint":"update_record","value":{"prim":"Pair","args":[{"bytes":"62616c6c732e74657a"},{"prim":"Pair","args":[{"prim":"Some","args":[{"string":"tz1dDQc4KsTHEFe3USc66Wti2pBatZ3UDbD4"}]},{"prim":"Pair","args":[{"string":"tz1WKygtstVY96oyc6Rmk945dMf33LeihgWT"},[]]}]}]}}`), DeffatedStorage: []byte(`{"prim":"Pair","args":[{"prim":"Pair","args":[{"bytes":"01535d971759846a1f2be8610e36f2db40fe8ce40800"},{"int":"1268"}]},{"bytes":"01ebb657570e494e8a7bd43ac3bf7cfd0267a32a9f00"}]}`), }, { - Kind: types.OperationKindTransaction, - Hash: "oocFt4vkkgQGfoRH54328cJUbDdWvj3x6KEs5Arm4XhqwwJmnJ8", - Internal: true, - Timestamp: timestamp, - Status: types.OperationStatusApplied, - Level: 1520888, - Nonce: newInt64Ptr(0), - Counter: 10671622, - Network: types.Mainnet, - ProtocolID: 4, - Entrypoint: "execute", + Kind: types.OperationKindTransaction, + Hash: "oocFt4vkkgQGfoRH54328cJUbDdWvj3x6KEs5Arm4XhqwwJmnJ8", + Internal: true, + Timestamp: timestamp, + Status: types.OperationStatusApplied, + Level: 1520888, + Nonce: newInt64Ptr(0), + Counter: 10671622, + Network: types.Mainnet, + ProtocolID: 4, + Entrypoint: types.NullString{ + Str: "execute", + Valid: true, + }, Initiator: "tz1WKygtstVY96oyc6Rmk945dMf33LeihgWT", Source: "KT1H1MqmUM4aK9i1833EBmYCCEfkbt6ZdSBc", Destination: "KT1GBZmSxmnKJXGMdMLbugPfLyUPmuLSMwKS", diff --git a/internal/parsers/operations/transaction.go b/internal/parsers/operations/transaction.go index f72e23e1b..f72ceab83 100644 --- a/internal/parsers/operations/transaction.go +++ b/internal/parsers/operations/transaction.go @@ -175,12 +175,13 @@ func (p Transaction) getEntrypoint(tx *operation.Operation) error { } if len(tx.Parameters) == 0 { - tx.Entrypoint = consts.DefaultEntrypoint - return nil + return tx.Entrypoint.Scan(consts.DefaultEntrypoint) } params := types.NewParameters(tx.Parameters) - tx.Entrypoint = params.Entrypoint + if err := tx.Entrypoint.Scan(params.Entrypoint); err != nil { + return err + } var tree ast.UntypedAST if err := json.Unmarshal(params.Value, &tree); err != nil { @@ -212,7 +213,6 @@ func (p Transaction) getEntrypoint(tx *operation.Operation) error { if node == nil { return nil } - tx.Entrypoint = entrypointName - return nil + return tx.Entrypoint.Scan(entrypointName) } diff --git a/internal/parsers/stacktrace/stacktrace.go b/internal/parsers/stacktrace/stacktrace.go index 4da09eaba..f641d4aef 100644 --- a/internal/parsers/stacktrace/stacktrace.go +++ b/internal/parsers/stacktrace/stacktrace.go @@ -7,12 +7,13 @@ import ( "github.com/baking-bad/bcdhub/internal/logger" "github.com/baking-bad/bcdhub/internal/models/operation" + "github.com/baking-bad/bcdhub/internal/models/types" ) // Item - type Item struct { ParentID int64 - Entrypoint string + Entrypoint types.NullString children []int64 source string @@ -23,15 +24,16 @@ type Item struct { // NewItem - func NewItem(operation operation.Operation, parentID int64) *Item { - return &Item{ + item := &Item{ ParentID: parentID, - Entrypoint: operation.Entrypoint, children: make([]int64, 0), source: operation.Source, destination: operation.Destination, contentIndex: operation.ContentIndex, nonce: operation.Nonce, + Entrypoint: operation.Entrypoint, } + return item } // GetID - @@ -41,7 +43,7 @@ func (sti *Item) GetID() int64 { // String - func (sti *Item) String() string { - s := sti.Entrypoint + s := sti.Entrypoint.String() if len(s) < 20 { s += strings.Repeat(" ", 20-len(s)) } diff --git a/internal/parsers/transfer/events.go b/internal/parsers/transfer/events.go index 8642af37b..ca57824a4 100644 --- a/internal/parsers/transfer/events.go +++ b/internal/parsers/transfer/events.go @@ -40,7 +40,7 @@ func (tokenEvents *TokenEvents) GetByOperation(operation operation.Operation) (t if event, ok := tokenEvents.events[ImplementationKey{ Address: operation.Destination, Network: operation.Network, - Entrypoint: operation.Entrypoint, + Entrypoint: operation.Entrypoint.String(), Name: tokenbalance.SingleAssetBalanceUpdates, }]; ok { return event, tokenbalance.SingleAssetBalanceUpdates, ok @@ -49,7 +49,7 @@ func (tokenEvents *TokenEvents) GetByOperation(operation operation.Operation) (t event, ok := tokenEvents.events[ImplementationKey{ Address: operation.Destination, Network: operation.Network, - Entrypoint: operation.Entrypoint, + Entrypoint: operation.Entrypoint.String(), Name: tokenbalance.MultiAssetBalanceUpdates, }] if ok { diff --git a/internal/parsers/transfer/transfer.go b/internal/parsers/transfer/transfer.go index a67ca539f..3f0fa485e 100644 --- a/internal/parsers/transfer/transfer.go +++ b/internal/parsers/transfer/transfer.go @@ -131,7 +131,7 @@ func (p *Parser) executeEvents(impl tzip.EventImplementation, name, protocol str } switch { - case impl.MichelsonParameterEvent != nil && impl.MichelsonParameterEvent.Is(operation.Entrypoint): + case impl.MichelsonParameterEvent != nil && impl.MichelsonParameterEvent.Is(operation.Entrypoint.String()): parameter, err := operation.AST.ParameterType() if err != nil { return err @@ -142,13 +142,13 @@ func (p *Parser) executeEvents(impl tzip.EventImplementation, name, protocol str return err } ctx.Parameters = subTree - ctx.Entrypoint = operation.Entrypoint + ctx.Entrypoint = operation.Entrypoint.String() event, err = events.NewMichelsonParameter(impl, name) if err != nil { return err } return p.makeTransfersFromBalanceEvents(event, ctx, operation, true) - case impl.MichelsonExtendedStorageEvent != nil && impl.MichelsonExtendedStorageEvent.Is(operation.Entrypoint): + case impl.MichelsonExtendedStorageEvent != nil && impl.MichelsonExtendedStorageEvent.Is(operation.Entrypoint.String()): storage, err := operation.AST.StorageType() if err != nil { return err diff --git a/internal/postgres/contract/storage.go b/internal/postgres/contract/storage.go index 2fc988ff2..9747c3d23 100644 --- a/internal/postgres/contract/storage.go +++ b/internal/postgres/contract/storage.go @@ -95,8 +95,8 @@ func (storage *Storage) GetProjectsLastContract(c contract.Contract, size, offse Where("encode(fingerprint_parameter, 'hex') = ?", params). Where("encode(fingerprint_storage, 'hex') = ?", s), ) - if c.Manager != "" { - subQuery.Or("manager = ?", c.Manager) + if c.Manager.Valid { + subQuery.Or("manager = ?", c.Manager.String()) } if c.Language != "unknown" { subQuery.Or("language = ?", c.Language) diff --git a/internal/postgres/core/histogram.go b/internal/postgres/core/histogram.go index 800aef384..d001e17fd 100644 --- a/internal/postgres/core/histogram.go +++ b/internal/postgres/core/histogram.go @@ -26,16 +26,16 @@ const ( histogramRequestTemplate = ` with f as ( select generate_series( - date_trunc('%s', %s), - date_trunc('%s', now()), - '1 %s'::interval + date_trunc(?, %s), + date_trunc(?, now()), + ?::interval ) as val ) select extract(epoch from f.val), %s from f - left join %s on date_trunc('%s', %s.timestamp) = f.val %s + left join %s on date_trunc(?, %s.timestamp) = f.val %s where %s.id is not null group by 1 order by date_part @@ -113,9 +113,11 @@ func (res HistogramResponse) toFloat64() []float64 { // GetDateHistogram - func (p *Postgres) GetDateHistogram(period string, opts ...models.HistogramOption) ([][]float64, error) { - ctx := models.HistogramContext{ - Period: period, + if err := ValidateHistogramPeriod(period); err != nil { + return nil, err } + + ctx := models.HistogramContext{} for _, opt := range opts { opt(&ctx) } @@ -125,8 +127,10 @@ func (p *Postgres) GetDateHistogram(period string, opts ...models.HistogramOptio return nil, err } + periodName := name(period) + var res []HistogramResponse - if err := p.DB.Raw(req).Scan(&res).Error; err != nil { + if err := p.DB.Raw(req, periodName, periodName, fmt.Sprintf("1 %s", periodName), periodName).Scan(&res).Error; err != nil { return nil, err } hist := make([][]float64, 0, len(res)) @@ -151,17 +155,12 @@ func (p *Postgres) GetCachedHistogram(period, name, network string) ([][]float64 } func getRequest(period, table, f, conditions string) (string, error) { - if err := ValidateHistogramPeriod(period); err != nil { - return "", err - } - if !helpers.StringInArray(table, []string{models.DocContracts, models.DocOperations, models.DocTransfers}) { return "", errors.Errorf("Invalid table: %s", table) } from := GetHistogramInterval(period) - periodName := name(period) - return fmt.Sprintf(histogramRequestTemplate, periodName, from, periodName, periodName, f, table, periodName, table, conditions, table), nil + return fmt.Sprintf(histogramRequestTemplate, from, f, table, table, conditions, table), nil } // ValidateHistogramPeriod - diff --git a/internal/search/contract.go b/internal/search/contract.go index 9eb543382..b5baa660e 100644 --- a/internal/search/contract.go +++ b/internal/search/contract.go @@ -92,14 +92,14 @@ func (c *Contract) Prepare(model models.Model) { c.Address = cont.Address c.Annotations = cont.Annotations - c.Delegate = cont.Delegate + c.Delegate = cont.Delegate.String() c.Entrypoints = cont.Entrypoints c.FailStrings = cont.FailStrings c.Hardcoded = cont.Hardcoded c.Hash = cont.Hash c.Language = cont.Language c.Level = cont.Level - c.Manager = cont.Manager + c.Manager = cont.Manager.String() c.Network = cont.Network.String() c.ProjectID = cont.ProjectID c.Tags = cont.Tags.ToArray() diff --git a/internal/search/operation.go b/internal/search/operation.go index 3abff9acc..0024cce61 100644 --- a/internal/search/operation.go +++ b/internal/search/operation.go @@ -90,7 +90,9 @@ func (o *Operation) Prepare(model models.Model) { o.ID = helpers.GenerateID() o.Destination = op.Destination o.DestinationAlias = op.Destination - o.Entrypoint = op.Entrypoint + if op.Entrypoint.Valid { + o.Entrypoint = op.Entrypoint.String() + } o.Hash = op.Hash o.Initiator = op.Initiator o.Internal = op.Internal diff --git a/scripts/migration/main.go b/scripts/migration/main.go index d410a2910..2380e75ec 100644 --- a/scripts/migration/main.go +++ b/scripts/migration/main.go @@ -36,6 +36,7 @@ var migrationsList = []migrations.Migration{ &migrations.TagsToInt{}, &migrations.DropAliasesColumns{}, &migrations.FixLostSearchContracts{}, + migrations.NewNullableFields(1000), } func main() { diff --git a/scripts/migration/migrations/nullable.go b/scripts/migration/migrations/nullable.go new file mode 100644 index 000000000..b9ee86981 --- /dev/null +++ b/scripts/migration/migrations/nullable.go @@ -0,0 +1,155 @@ +package migrations + +import ( + "fmt" + + "github.com/baking-bad/bcdhub/internal/config" + "github.com/baking-bad/bcdhub/internal/logger" + "github.com/baking-bad/bcdhub/internal/models/contract" + "github.com/baking-bad/bcdhub/internal/models/operation" + "github.com/baking-bad/bcdhub/internal/models/transfer" + "gorm.io/gorm" +) + +// NullableFields - +type NullableFields struct { + limit int +} + +// NewNullableFields - +func NewNullableFields(limit int) *NullableFields { + return &NullableFields{limit} +} + +// Key - +func (m *NullableFields) Key() string { + return "nullable_fields" +} + +// Description - +func (m *NullableFields) Description() string { + return "set some fields to nullable type" +} + +// Do - migrate function +func (m *NullableFields) Do(ctx *config.Context) error { + if m.limit == 0 { + m.limit = 10000 + } + return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error { + if err := m.migrateContracts(tx); err != nil { + return err + } + if err := m.migrateOperations(tx); err != nil { + return err + } + if err := m.migrateTransfers(tx); err != nil { + return err + } + return nil + }) +} + +func (m *NullableFields) migrateContracts(tx *gorm.DB) error { + logger.Info().Msg("processing contracts...") + + var end bool + var offset int + for !end { + var contracts []contract.Contract + if err := tx.Model(&contract.Contract{}). + Where("delegate = ''").Or("manager = ''"). + Limit(m.limit). + Offset(offset). + Find(&contracts).Error; err != nil { + return err + } + + for i := range contracts { + fields := make(map[string]interface{}) + if contracts[i].Manager.Str == "" && contracts[i].Manager.Valid { + contracts[i].Manager.Valid = false + fields["manager"] = contracts[i].Manager + } + if contracts[i].Delegate.Str == "" && contracts[i].Delegate.Valid { + contracts[i].Delegate.Valid = false + fields["delegate"] = contracts[i].Delegate + } + if len(fields) > 0 { + if err := tx.Model(&contracts[i]).Updates(fields).Error; err != nil { + return err + } + } + } + + offset += len(contracts) + end = len(contracts) < m.limit + fmt.Printf("processed %d", offset) + } + + return nil +} + +func (m *NullableFields) migrateOperations(tx *gorm.DB) error { + logger.Info().Msg("processing operations...") + + var end bool + var offset int + for !end { + var operations []operation.Operation + if err := tx.Model(&operation.Operation{}). + Where("entrypoint = ''"). + Limit(m.limit). + Offset(offset). + Find(&operations).Error; err != nil { + return err + } + + for i := range operations { + if operations[i].Entrypoint.Str == "" && operations[i].Entrypoint.Valid { + operations[i].Entrypoint.Valid = false + if err := tx.Model(&operations[i]).Update("entrypoint", operations[i].Entrypoint).Error; err != nil { + return err + } + } + } + + offset += len(operations) + end = len(operations) < m.limit + fmt.Printf("processed %d\r", offset) + } + + return nil +} + +func (m *NullableFields) migrateTransfers(tx *gorm.DB) error { + logger.Info().Msg("processing transfers...") + + var end bool + var offset int + for !end { + var transfers []transfer.Transfer + if err := tx.Model(&transfer.Transfer{}). + Where("parent = ''"). + Limit(m.limit). + Offset(offset). + Find(&transfers).Error; err != nil { + return err + } + + for i := range transfers { + if transfers[i].Parent.Str == "" && transfers[i].Parent.Valid { + transfers[i].Parent.Valid = false + if err := tx.Model(&transfers[i]).Update("parent", transfers[i].Parent).Error; err != nil { + return err + } + } + } + + offset += len(transfers) + end = len(transfers) < m.limit + fmt.Printf("processed %d\r", offset) + } + + return nil +}