From 863090f8eff3fc6521a4a50b25ac67a8dfd5a218 Mon Sep 17 00:00:00 2001 From: Andrew Low Date: Mon, 16 Jan 2023 19:23:18 -0800 Subject: [PATCH] address comments --- analyzer/api.go | 22 -------- analyzer/consensus/consensus.go | 52 ++++++++----------- api/spec/v1.yaml | 4 +- api/v1/client.go | 11 ++-- api/v1/types/openapi.gen.go | 10 ++-- api/v1/types/util.go | 22 ++++++++ storage/client/api.go | 2 +- storage/client/client.go | 4 +- storage/client/queries.go | 2 +- .../migrations/01_oasis_3_consensus.up.sql | 6 +-- .../05_negative_balance_debug.up.sql | 2 - 11 files changed, 64 insertions(+), 73 deletions(-) create mode 100644 api/v1/types/util.go delete mode 100644 storage/migrations/05_negative_balance_debug.up.sql diff --git a/analyzer/api.go b/analyzer/api.go index 909bf8464..3d7ef75e6 100644 --- a/analyzer/api.go +++ b/analyzer/api.go @@ -6,7 +6,6 @@ import ( oasisConfig "github.com/oasisprotocol/oasis-sdk/client-sdk/go/config" - apiTypes "github.com/oasisprotocol/oasis-indexer/api/v1/types" "github.com/oasisprotocol/oasis-indexer/storage" ) @@ -82,27 +81,6 @@ type RoundRange struct { To uint64 } -var StringToEvent map[string]apiTypes.ConsensusEventType = map[string]apiTypes.ConsensusEventType{ - "staking.transfer": apiTypes.ConsensusEventTypeStakingTransfer, - "staking.burn": apiTypes.ConsensusEventTypeStakingBurn, - "staking.escrow.add": apiTypes.ConsensusEventTypeStakingEscrowAdd, - "staking.escrow.take": apiTypes.ConsensusEventTypeStakingEscrowTake, - "staking.escrow.debonding_start": apiTypes.ConsensusEventTypeStakingEscrowDebondingStart, - "staking.escrow.reclaim": apiTypes.ConsensusEventTypeStakingEscrowReclaim, - "staking.allowance_change": apiTypes.ConsensusEventTypeStakingAllowanceChange, - "registry.runtime": apiTypes.ConsensusEventTypeRegistryRuntime, - "registry.entity": apiTypes.ConsensusEventTypeRegistryEntity, - "registry.node": apiTypes.ConsensusEventTypeRegistryNode, - "registry.node_unfrozen": apiTypes.ConsensusEventTypeRegistryNodeUnfrozen, - "roothash.executor_committed": apiTypes.ConsensusEventTypeRoothashExecutorCommitted, - "roothash.execution_discrepancy_detected": apiTypes.ConsensusEventTypeRoothashExecutionDiscrepancy, - "roothash.finalized": apiTypes.ConsensusEventTypeRoothashFinalized, - "governance.proposal_submitted": apiTypes.ConsensusEventTypeGovernanceProposalSubmitted, - "governance.proposal_executed": apiTypes.ConsensusEventTypeGovernanceProposalExecuted, - "governance.proposal_finalized": apiTypes.ConsensusEventTypeGovernanceProposalFinalized, - "governance.vote": apiTypes.ConsensusEventTypeGovernanceVote, -} - // ChainID is the ID of a chain. type ChainID string diff --git a/analyzer/consensus/consensus.go b/analyzer/consensus/consensus.go index 216788817..21365840c 100644 --- a/analyzer/consensus/consensus.go +++ b/analyzer/consensus/consensus.go @@ -464,12 +464,12 @@ func (m *Main) queueTxEventInserts(batch *storage.QueryBatch, data *storage.Cons for i := 0; i < len(data.Results); i++ { var txAccounts []staking.Address for j := 0; j < len(data.Results[i].Events); j++ { - eventData, err := extractEventData(data.Results[i].Events[j]) + eventData, err := m.extractEventData(data.Results[i].Events[j]) if err != nil { return err } txAccounts = append(txAccounts, eventData.relatedAddresses...) - accounts := m.extractUniqueAddresses(eventData.relatedAddresses) + accounts := extractUniqueAddresses(eventData.relatedAddresses) body, err := json.Marshal(eventData.body) if err != nil { return err @@ -484,7 +484,7 @@ func (m *Main) queueTxEventInserts(batch *storage.QueryBatch, data *storage.Cons accounts, ) } - uniqueTxAccounts := m.extractUniqueAddresses(txAccounts) + uniqueTxAccounts := extractUniqueAddresses(txAccounts) for _, addr := range uniqueTxAccounts { batch.Queue(accountRelatedTransactionInsertQuery, addr, @@ -963,14 +963,13 @@ func (m *Main) queueGovernanceEventInserts(batch *storage.QueryBatch, data *stor } func (m *Main) queueSingleEventInserts(batch *storage.QueryBatch, eventData *parsedEvent, height int64) error { - eventInsertQuery := m.qf.ConsensusEventInsertQuery() - accounts := m.extractUniqueAddresses(eventData.relatedAddresses) + accounts := extractUniqueAddresses(eventData.relatedAddresses) body, err := json.Marshal(eventData.body) if err != nil { return err } - batch.Queue(eventInsertQuery, + batch.Queue(m.qf.ConsensusEventInsertQuery(), string(eventData.ty), string(body), height, @@ -982,7 +981,7 @@ func (m *Main) queueSingleEventInserts(batch *storage.QueryBatch, eventData *par return nil } -func (m *Main) extractUniqueAddresses(accounts []staking.Address) []string { +func extractUniqueAddresses(accounts []staking.Address) []string { var uniqueAccounts []string seen := make(map[string]struct{}) for _, addr := range accounts { @@ -998,7 +997,7 @@ func (m *Main) extractUniqueAddresses(accounts []staking.Address) []string { } // extractEventData extracts the type, the body (JSON-serialized), and the related accounts of an event. -func extractEventData(event *results.Event) (*parsedEvent, error) { +func (m *Main) extractEventData(event *results.Event) (*parsedEvent, error) { switch e := event; { case e.Staking != nil: return extractStakingEvent(event.Staking) @@ -1010,6 +1009,8 @@ func extractEventData(event *results.Event) (*parsedEvent, error) { return extractGovernanceEvent(event.Governance) } + m.logger.Error("cannot infer consensus event type from event struct", "event", event) + return &parsedEvent{ ty: EventType("unknown"), body: []byte{}, @@ -1019,13 +1020,12 @@ func extractEventData(event *results.Event) (*parsedEvent, error) { func extractGovernanceEvent(event *governance.Event) (*parsedEvent, error) { var eventData parsedEvent - var accounts []staking.Address var err error switch event := event; { case event.ProposalSubmitted != nil: eventData.ty = apiTypes.ConsensusEventTypeGovernanceProposalSubmitted eventData.body = event.ProposalSubmitted - accounts = append(accounts, event.ProposalSubmitted.Submitter) + eventData.relatedAddresses = []staking.Address{event.ProposalSubmitted.Submitter} case event.ProposalExecuted != nil: eventData.ty = apiTypes.ConsensusEventTypeGovernanceProposalExecuted eventData.body = event.ProposalExecuted @@ -1035,24 +1035,22 @@ func extractGovernanceEvent(event *governance.Event) (*parsedEvent, error) { case event.Vote != nil: eventData.ty = apiTypes.ConsensusEventTypeGovernanceVote eventData.body = event.Vote - accounts = append(accounts, event.Vote.Submitter) + eventData.relatedAddresses = []staking.Address{event.Vote.Submitter} default: err = fmt.Errorf("unsupported registry event type: %#v", event) } - eventData.relatedAddresses = accounts return &eventData, err } func extractRootHashEvent(event *roothash.Event) (*parsedEvent, error) { var eventData parsedEvent - var accounts []staking.Address var err error switch event := event; { case event.ExecutorCommitted != nil: eventData.ty = apiTypes.ConsensusEventTypeRoothashExecutorCommitted nodeAddr := staking.NewAddress(event.ExecutorCommitted.Commit.NodeID) eventData.body = event.ExecutorCommitted - accounts = append(accounts, nodeAddr) + eventData.relatedAddresses = []staking.Address{nodeAddr} case event.ExecutionDiscrepancyDetected != nil: eventData.ty = apiTypes.ConsensusEventTypeRoothashExecutionDiscrepancy eventData.body = event.ExecutionDiscrepancyDetected @@ -1062,13 +1060,11 @@ func extractRootHashEvent(event *roothash.Event) (*parsedEvent, error) { default: err = fmt.Errorf("unsupported registry event type: %#v", event) } - eventData.relatedAddresses = accounts return &eventData, err } func extractRegistryEvent(event *registry.Event) (*parsedEvent, error) { var eventData parsedEvent - var accounts []staking.Address var err error switch event := event; { case event.RuntimeEvent != nil: @@ -1078,68 +1074,66 @@ func extractRegistryEvent(event *registry.Event) (*parsedEvent, error) { eventData.ty = apiTypes.ConsensusEventTypeRegistryEntity eventData.body = event.EntityEvent addr := staking.NewAddress(event.EntityEvent.Entity.ID) - accounts = append(accounts, addr) + accounts := []staking.Address{addr} for _, node := range event.EntityEvent.Entity.Nodes { nodeAddr := staking.NewAddress(node) accounts = append(accounts, nodeAddr) } + eventData.relatedAddresses = accounts case event.NodeEvent != nil: eventData.ty = apiTypes.ConsensusEventTypeRegistryNode eventData.body = event.NodeEvent nodeAddr := staking.NewAddress(event.NodeEvent.Node.EntityID) entityAddr := staking.NewAddress(event.NodeEvent.Node.ID) - accounts = append(accounts, nodeAddr, entityAddr) + eventData.relatedAddresses = []staking.Address{nodeAddr, entityAddr} case event.NodeUnfrozenEvent != nil: eventData.ty = apiTypes.ConsensusEventTypeRegistryNodeUnfrozen eventData.body = event.NodeUnfrozenEvent nodeAddr := staking.NewAddress(event.NodeUnfrozenEvent.NodeID) - accounts = append(accounts, nodeAddr) + eventData.relatedAddresses = []staking.Address{nodeAddr} default: err = fmt.Errorf("unsupported registry event type: %#v", event) } - eventData.relatedAddresses = accounts return &eventData, err } func extractStakingEvent(event *staking.Event) (*parsedEvent, error) { var eventData parsedEvent - var accounts []staking.Address var err error switch event := event; { case event.Transfer != nil: eventData.ty = apiTypes.ConsensusEventTypeStakingTransfer eventData.body = event.Transfer - accounts = append(accounts, event.Transfer.From, event.Transfer.To) + eventData.relatedAddresses = []staking.Address{event.Transfer.From, event.Transfer.To} case event.Burn != nil: eventData.ty = apiTypes.ConsensusEventTypeStakingBurn eventData.body = event.Burn - accounts = append(accounts, event.Burn.Owner) + eventData.relatedAddresses = []staking.Address{event.Burn.Owner} case event.Escrow != nil: switch t := event.Escrow; { case t.Add != nil: eventData.ty = apiTypes.ConsensusEventTypeStakingEscrowAdd eventData.body = event.Escrow.Add - accounts = append(accounts, t.Add.Owner, t.Add.Escrow) + eventData.relatedAddresses = []staking.Address{t.Add.Owner, t.Add.Escrow} case t.Take != nil: eventData.ty = apiTypes.ConsensusEventTypeStakingEscrowTake eventData.body = event.Escrow.Take - accounts = append(accounts, t.Take.Owner) + eventData.relatedAddresses = []staking.Address{t.Take.Owner} case t.DebondingStart != nil: eventData.ty = apiTypes.ConsensusEventTypeStakingEscrowDebondingStart eventData.body = event.Escrow.DebondingStart - accounts = append(accounts, t.DebondingStart.Owner, t.DebondingStart.Escrow) + eventData.relatedAddresses = []staking.Address{t.DebondingStart.Owner, t.DebondingStart.Escrow} case t.Reclaim != nil: eventData.ty = apiTypes.ConsensusEventTypeStakingEscrowReclaim eventData.body = event.Escrow.Reclaim - accounts = append(accounts, t.Reclaim.Owner, t.Reclaim.Escrow) + eventData.relatedAddresses = []staking.Address{t.Reclaim.Owner, t.Reclaim.Escrow} } case event.AllowanceChange != nil: eventData.ty = apiTypes.ConsensusEventTypeStakingAllowanceChange eventData.body, err = json.Marshal(event.AllowanceChange) - accounts = append(accounts, event.AllowanceChange.Owner, event.AllowanceChange.Beneficiary) + eventData.relatedAddresses = []staking.Address{event.AllowanceChange.Owner, event.AllowanceChange.Beneficiary} default: err = fmt.Errorf("unsupported registry event type: %#v", event) } - eventData.relatedAddresses = accounts return &eventData, err } diff --git a/api/spec/v1.yaml b/api/spec/v1.yaml index eb6beccc5..709c66cc3 100644 --- a/api/spec/v1.yaml +++ b/api/spec/v1.yaml @@ -280,7 +280,7 @@ paths: - *limit - *offset - in: query - name: height + name: block schema: type: integer format: int64 @@ -1065,7 +1065,7 @@ components: type: object required: [height, type, body] properties: - height: + block: type: integer format: int64 description: The block height at which this event was generated. diff --git a/api/v1/client.go b/api/v1/client.go index ea0cea384..55c3def67 100644 --- a/api/v1/client.go +++ b/api/v1/client.go @@ -14,7 +14,6 @@ import ( governance "github.com/oasisprotocol/oasis-core/go/governance/api" staking "github.com/oasisprotocol/oasis-core/go/staking/api" - analyzerApi "github.com/oasisprotocol/oasis-indexer/analyzer" apiCommon "github.com/oasisprotocol/oasis-indexer/api/common" apiTypes "github.com/oasisprotocol/oasis-indexer/api/v1/types" "github.com/oasisprotocol/oasis-indexer/common" @@ -81,7 +80,7 @@ func validateConsensusAddress(param string) (*staking.Address, error) { // validateEventType parses a consensus event type url parameter. func validateEventType(param string) (*string, error) { - _, ok := analyzerApi.StringToEvent[param] + _, ok := apiTypes.StringToEvent[param] if ok { return ¶m, nil } @@ -257,16 +256,16 @@ func (c *storageClient) Events(ctx context.Context, r *http.Request) (*apiTypes. var q storage.EventsRequest params := r.URL.Query() - if v := params.Get("height"); v != "" { - height, err := validateInt64(v) + if v := params.Get("block"); v != "" { + block, err := validateInt64(v) if err != nil { return nil, apiCommon.ErrBadRequest } - q.Height = &height + q.Block = &block } if v := params.Get("tx_index"); v != "" { // If tx_index is provided, height must also be provided. - if q.Height == nil { + if q.Block == nil { return nil, apiCommon.ErrBadRequest } diff --git a/api/v1/types/openapi.gen.go b/api/v1/types/openapi.gen.go index 1934eb83a..cf8a14b9e 100644 --- a/api/v1/types/openapi.gen.go +++ b/api/v1/types/openapi.gen.go @@ -189,15 +189,15 @@ type BlockList struct { // ConsensusEvent An event emitted by the consensus layer. type ConsensusEvent struct { + // Block The block height at which this event was generated. + Block *int64 `json:"block,omitempty"` + // Body The event contents. This spec does not encode the many possible types; // instead, see [the Go API](https://pkg.go.dev/github.com/oasisprotocol/oasis-core/go/consensus/api/transaction/results#Event) of oasis-core. // This object will conform to one of the `*Event` types two levels down // the hierarchy, e.g. `TransferEvent` from `Event > staking.Event > TransferEvent` Body map[string]interface{} `json:"body"` - // Height The block height at which this event was generated. - Height int64 `json:"height"` - // TxHash Hash of this event's originating transaction. // Absent if the event did not originate from a transaction. TxHash *string `json:"tx_hash"` @@ -747,8 +747,8 @@ type GetConsensusEventsParams struct { // Offset The number of items to skip before starting to collect the result set. Offset *uint64 `form:"offset,omitempty" json:"offset,omitempty"` - // Height A filter on block height. - Height *int64 `form:"height,omitempty" json:"height,omitempty"` + // Block A filter on block height. + Block *int64 `form:"block,omitempty" json:"block,omitempty"` // TxIndex A filter on transaction index. The returned events all need to originate // from a transaction that appeared in `tx_index`-th position in the block. diff --git a/api/v1/types/util.go b/api/v1/types/util.go new file mode 100644 index 000000000..c40322a1c --- /dev/null +++ b/api/v1/types/util.go @@ -0,0 +1,22 @@ +package types + +var StringToEvent map[string]ConsensusEventType = map[string]ConsensusEventType{ + "staking.transfer": ConsensusEventTypeStakingTransfer, + "staking.burn": ConsensusEventTypeStakingBurn, + "staking.escrow.add": ConsensusEventTypeStakingEscrowAdd, + "staking.escrow.take": ConsensusEventTypeStakingEscrowTake, + "staking.escrow.debonding_start": ConsensusEventTypeStakingEscrowDebondingStart, + "staking.escrow.reclaim": ConsensusEventTypeStakingEscrowReclaim, + "staking.allowance_change": ConsensusEventTypeStakingAllowanceChange, + "registry.runtime": ConsensusEventTypeRegistryRuntime, + "registry.entity": ConsensusEventTypeRegistryEntity, + "registry.node": ConsensusEventTypeRegistryNode, + "registry.node_unfrozen": ConsensusEventTypeRegistryNodeUnfrozen, + "roothash.executor_committed": ConsensusEventTypeRoothashExecutorCommitted, + "roothash.execution_discrepancy_detected": ConsensusEventTypeRoothashExecutionDiscrepancy, + "roothash.finalized": ConsensusEventTypeRoothashFinalized, + "governance.proposal_submitted": ConsensusEventTypeGovernanceProposalSubmitted, + "governance.proposal_executed": ConsensusEventTypeGovernanceProposalExecuted, + "governance.proposal_finalized": ConsensusEventTypeGovernanceProposalFinalized, + "governance.vote": ConsensusEventTypeGovernanceVote, +} diff --git a/storage/client/api.go b/storage/client/api.go index dbd8146ab..2cdc40bb5 100644 --- a/storage/client/api.go +++ b/storage/client/api.go @@ -37,7 +37,7 @@ type TransactionRequest struct { } type EventsRequest struct { - Height *int64 + Block *int64 TxIndex *int32 TxHash *string Rel *staking.Address diff --git a/storage/client/client.go b/storage/client/client.go index 6d6eeea1e..faa0ba4de 100644 --- a/storage/client/client.go +++ b/storage/client/client.go @@ -355,7 +355,7 @@ func (c *StorageClient) Events(ctx context.Context, r *EventsRequest, p *apiComm rows, err = c.db.Query( ctx, qf.EventsQuery(), - r.Height, + r.Block, r.TxIndex, r.TxHash, r.Type, @@ -379,7 +379,7 @@ func (c *StorageClient) Events(ctx context.Context, r *EventsRequest, p *apiComm for rows.Next() { var e Event - if err := rows.Scan(&e.Height, &e.TxIndex, &e.TxHash, &e.Type, &e.Body); err != nil { + if err := rows.Scan(&e.Block, &e.TxIndex, &e.TxHash, &e.Type, &e.Body); err != nil { c.logger.Info("query failed", "err", err.Error(), ) diff --git a/storage/client/queries.go b/storage/client/queries.go index 269eb6f9a..6e16d2a16 100644 --- a/storage/client/queries.go +++ b/storage/client/queries.go @@ -86,7 +86,7 @@ func (qf QueryFactory) EventsQuery() string { ($2::integer IS NULL OR tx_index = $2::integer) AND ($3::text IS NULL OR tx_hash = $3::text) AND ($4::text IS NULL OR type = $4::text) AND - ($5::text IS NULL OR $5::text = ANY(related_accounts)) + ($5::text IS NULL OR ARRAY[$5::text] <@ related_accounts) ORDER BY tx_block DESC, tx_index LIMIT $6::bigint OFFSET $7::bigint`, qf.chainID) diff --git a/storage/migrations/01_oasis_3_consensus.up.sql b/storage/migrations/01_oasis_3_consensus.up.sql index e210697b5..4842d96a6 100644 --- a/storage/migrations/01_oasis_3_consensus.up.sql +++ b/storage/migrations/01_oasis_3_consensus.up.sql @@ -35,7 +35,7 @@ CREATE TABLE oasis_3.blocks CREATE TABLE oasis_3.transactions ( - block UINT63 NOT NULL REFERENCES oasis_3.blocks(height) DEFERRABLE INITIALLY DEFERRED, -- TODO: rename to height + block UINT63 NOT NULL REFERENCES oasis_3.blocks(height) DEFERRABLE INITIALLY DEFERRED, tx_index UINT31 NOT NULL, tx_hash HEX64 NOT NULL, @@ -62,7 +62,7 @@ CREATE INDEX ix_transactions_tx_hash ON oasis_3.transactions (tx_hash); CREATE TABLE oasis_3.events ( - tx_block UINT63 NOT NULL, -- TODO: rename to height + tx_block UINT63 NOT NULL, tx_index UINT31, type TEXT NOT NULL, -- Enum with many values, see https://github.com/oasisprotocol/oasis-indexer/blob/89b68717205809b491d7926533d096444611bd6b/analyzer/api.go#L171-L171 @@ -263,7 +263,7 @@ CREATE TABLE oasis_3.accounts_related_transactions tx_index UINT31 NOT NULL, FOREIGN KEY (tx_block, tx_index) REFERENCES oasis_3.transactions(block, tx_index) DEFERRABLE INITIALLY DEFERRED ); -CREATE INDEX ix_accounts_related_transactions_address_block_index ON oasis_3.accounts_related_transactions (account_address, tx_block, tx_index); +CREATE INDEX ix_accounts_related_transactions_address_block_index ON oasis_3.accounts_related_transactions (account_address); -- Indexing Progress Management CREATE TABLE oasis_3.processed_blocks diff --git a/storage/migrations/05_negative_balance_debug.up.sql b/storage/migrations/05_negative_balance_debug.up.sql deleted file mode 100644 index 45f5b8f49..000000000 --- a/storage/migrations/05_negative_balance_debug.up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE oasis_3.accounts - ADD CONSTRAINT positive_balance CHECK (address = 'oasis1qqnv3peudzvekhulf8v3ht29z4cthkhy7gkxmph5' OR general_balance >= 0);