Merge pull request #356 from oasisprotocol/mitjat/cobalt-consensus

analyzer/consensus: Full Cobalt support. Internal types for node responses.

cmd/common/common.go

@@ -6,6 +6,8 @@ import (
 	"io"
 	"os"
 
+	coreLogging "github.com/oasisprotocol/oasis-core/go/common/logging"
+
 	"github.com/oasisprotocol/oasis-indexer/config"
 	"github.com/oasisprotocol/oasis-indexer/log"
 	"github.com/oasisprotocol/oasis-indexer/metrics"
@@ -20,6 +22,10 @@ func Init(cfg *config.Config) error {
 	var w io.Writer = os.Stdout
 	format := log.FmtJSON
 	level := log.LevelDebug
+	coreFormat := coreLogging.FmtJSON   // For oasis-core.
+	coreLevel := coreLogging.LevelDebug // For oasis-core.
+
+	// Initialize oasis-indexer logging.
 	if cfg.Log != nil {
 		var err error
 		if w, err = getLoggingStream(cfg.Log); err != nil {
@@ -38,6 +44,12 @@ func Init(cfg *config.Config) error {
 	}
 	rootLogger = logger
 
+	// Initialize oasis-core logging. Useful for low-level gRPC issues.
+	if err := coreLogging.Initialize(w, coreFormat, coreLevel, nil); err != nil {
+		logger.Error("failed to initialize oasis-core logging", "err", err)
+		return err
+	}
+
 	// Initialize Prometheus service.
 	if cfg.Metrics != nil {
 		promServer, err := metrics.NewPullService(cfg.Metrics.PullEndpoint, rootLogger)

common/types.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 
 	"github.com/jackc/pgx/v5/pgtype"
+
+	"github.com/oasisprotocol/oasis-core/go/common/quantity"
 )
 
 // Arbitrary-precision integer. Wrapper around big.Int to allow for
@@ -35,6 +37,10 @@ func (b *BigInt) UnmarshalJSON(text []byte) error {
 	return b.Int.UnmarshalJSON([]byte(v))
 }
 
+func BigIntFromQuantity(q quantity.Quantity) BigInt {
+	return BigInt{*q.ToBigInt()}
+}
+
 // Implement NumericValuer interface for BigInt.
 func (b BigInt) NumericValue() (pgtype.Numeric, error) {
 	return pgtype.Numeric{Int: &b.Int, Exp: 0, NaN: false, Valid: true, InfinityModifier: pgtype.Finite}, nil

storage/api.go

@@ -10,15 +10,9 @@ import (
 	beacon "github.com/oasisprotocol/oasis-core/go/beacon/api"
 	"github.com/oasisprotocol/oasis-core/go/common"
 	consensus "github.com/oasisprotocol/oasis-core/go/consensus/api"
-	"github.com/oasisprotocol/oasis-core/go/consensus/api/transaction"
-	"github.com/oasisprotocol/oasis-core/go/consensus/api/transaction/results"
 	genesisAPI "github.com/oasisprotocol/oasis-core/go/genesis/api"
-	governance "github.com/oasisprotocol/oasis-core/go/governance/api"
-	registry "github.com/oasisprotocol/oasis-core/go/registry/api"
-	roothash "github.com/oasisprotocol/oasis-core/go/roothash/api"
 	"github.com/oasisprotocol/oasis-core/go/roothash/api/block"
-	scheduler "github.com/oasisprotocol/oasis-core/go/scheduler/api"
-	staking "github.com/oasisprotocol/oasis-core/go/staking/api"
+	"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi"
 	"github.com/oasisprotocol/oasis-sdk/client-sdk/go/client"
 	sdkTypes "github.com/oasisprotocol/oasis-sdk/client-sdk/go/types"
 )
@@ -149,10 +143,9 @@ type ConsensusAllData struct {
 type ConsensusBlockData struct {
 	Height int64
 
-	BlockHeader  *consensus.Block
-	Epoch        beacon.EpochTime
-	Transactions []*transaction.SignedTransaction
-	Results      []*results.Result
+	BlockHeader             *consensus.Block
+	Epoch                   beacon.EpochTime
+	TransactionsWithResults []nodeapi.TransactionWithResults
 }
 
 // BeaconData represents data for the random beacon at a given height.
@@ -170,11 +163,11 @@ type BeaconData struct {
 type RegistryData struct {
 	Height int64
 
-	Events             []*registry.Event
-	RuntimeEvents      []*registry.RuntimeEvent
-	EntityEvents       []*registry.EntityEvent
-	NodeEvents         []*registry.NodeEvent
-	NodeUnfrozenEvents []*registry.NodeUnfrozenEvent
+	Events             []nodeapi.Event
+	RuntimeEvents      []nodeapi.RuntimeEvent
+	EntityEvents       []nodeapi.EntityEvent
+	NodeEvents         []nodeapi.NodeEvent
+	NodeUnfrozenEvents []nodeapi.NodeUnfrozenEvent
 }
 
 // StakingData represents data for accounts at a given height.
@@ -185,26 +178,29 @@ type StakingData struct {
 	Height int64
 	Epoch  beacon.EpochTime
 
-	Events           []*staking.Event
-	Transfers        []*staking.TransferEvent
-	Burns            []*staking.BurnEvent
-	Escrows          []*staking.EscrowEvent
-	AllowanceChanges []*staking.AllowanceChangeEvent
+	Events                []nodeapi.Event
+	Transfers             []nodeapi.TransferEvent
+	Burns                 []nodeapi.BurnEvent
+	AddEscrows            []nodeapi.AddEscrowEvent
+	TakeEscrows           []nodeapi.TakeEscrowEvent
+	ReclaimEscrows        []nodeapi.ReclaimEscrowEvent
+	DebondingStartEscrows []nodeapi.DebondingStartEscrowEvent
+	AllowanceChanges      []nodeapi.AllowanceChangeEvent
 }
 
 // RootHashData represents data for runtime processing at a given height.
 type RootHashData struct {
 	Height int64
 
-	Events []*roothash.Event
+	Events []nodeapi.Event
 }
 
 // SchedulerData represents data for elected committees and validators at a given height.
 type SchedulerData struct {
 	Height int64
 
-	Validators []*scheduler.Validator
-	Committees map[common.Namespace][]*scheduler.Committee
+	Validators []nodeapi.Validator
+	Committees map[common.Namespace][]nodeapi.Committee
 }
 
 // GovernanceData represents governance data for proposals at a given height.
@@ -214,11 +210,11 @@ type SchedulerData struct {
 type GovernanceData struct {
 	Height int64
 
-	Events                []*governance.Event
-	ProposalSubmissions   []*governance.Proposal
-	ProposalExecutions    []*governance.ProposalExecutedEvent
-	ProposalFinalizations []*governance.Proposal
-	Votes                 []*governance.VoteEvent
+	Events                []nodeapi.Event
+	ProposalSubmissions   []nodeapi.Proposal
+	ProposalExecutions    []nodeapi.ProposalExecutedEvent
+	ProposalFinalizations []nodeapi.Proposal
+	Votes                 []nodeapi.VoteEvent
 }
 
 // RuntimeSourceStorage defines an interface for retrieving raw block data

storage/oasis/consensus.go

@@ -6,20 +6,22 @@ import (
 
 	beaconAPI "github.com/oasisprotocol/oasis-core/go/beacon/api"
 	"github.com/oasisprotocol/oasis-core/go/common"
-	"github.com/oasisprotocol/oasis-core/go/common/cbor"
-	"github.com/oasisprotocol/oasis-core/go/consensus/api/transaction"
 	genesisAPI "github.com/oasisprotocol/oasis-core/go/genesis/api"
-	governanceAPI "github.com/oasisprotocol/oasis-core/go/governance/api"
-	registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api"
-	schedulerAPI "github.com/oasisprotocol/oasis-core/go/scheduler/api"
-	stakingAPI "github.com/oasisprotocol/oasis-core/go/staking/api"
 	"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi"
 	config "github.com/oasisprotocol/oasis-sdk/client-sdk/go/config"
 
 	"github.com/oasisprotocol/oasis-indexer/storage"
 )
 
-// ConsensusClient is a client to the consensus backends.
+// ConsensusClient is a client to the consensus methods/data of oasis node. It
+// differs from the nodeapi.ConsensusApiLite in that:
+//   - Its methods may collect data using multiple RPCs each.
+//   - The return types make no effort to closely resemble oasis-core types
+//     in structure. Instead, they are structured in a way that is most convenient
+//     for the analyzer.
+//     TODO: The benefits of this are miniscule, and introduce considerable
+//     boilerplate. Consider removing most types from this package, and
+//     using nodeapi types directly.
 type ConsensusClient struct {
 	nodeApi nodeapi.ConsensusApiLite
 	network *config.Network
@@ -105,21 +107,11 @@ func (cc *ConsensusClient) BlockData(ctx context.Context, height int64) (*storag
 		return nil, err
 	}
 
-	transactions := make([]*transaction.SignedTransaction, 0, len(transactionsWithResults.Transactions))
-	for _, bytes := range transactionsWithResults.Transactions {
-		var transaction transaction.SignedTransaction
-		if err := cbor.Unmarshal(bytes, &transaction); err != nil {
-			return nil, err
-		}
-		transactions = append(transactions, &transaction)
-	}
-
 	return &storage.ConsensusBlockData{
-		Height:       height,
-		BlockHeader:  block,
-		Epoch:        epoch,
-		Transactions: transactions,
-		Results:      transactionsWithResults.Results,
+		Height:                  height,
+		BlockHeader:             block,
+		Epoch:                   epoch,
+		TransactionsWithResults: transactionsWithResults,
 	}, nil
 }
 
@@ -151,21 +143,21 @@ func (cc *ConsensusClient) RegistryData(ctx context.Context, height int64) (*sto
 		return nil, err
 	}
 
-	var runtimeEvents []*registryAPI.RuntimeEvent
-	var entityEvents []*registryAPI.EntityEvent
-	var nodeEvents []*registryAPI.NodeEvent
-	var nodeUnfrozenEvents []*registryAPI.NodeUnfrozenEvent
+	var runtimeEvents []nodeapi.RuntimeEvent
+	var entityEvents []nodeapi.EntityEvent
+	var nodeEvents []nodeapi.NodeEvent
+	var nodeUnfrozenEvents []nodeapi.NodeUnfrozenEvent
 
 	for _, event := range events {
 		switch e := event; {
-		case e.RuntimeEvent != nil:
-			runtimeEvents = append(runtimeEvents, e.RuntimeEvent)
-		case e.EntityEvent != nil:
-			entityEvents = append(entityEvents, e.EntityEvent)
-		case e.NodeEvent != nil:
-			nodeEvents = append(nodeEvents, e.NodeEvent)
-		case e.NodeUnfrozenEvent != nil:
-			nodeUnfrozenEvents = append(nodeUnfrozenEvents, e.NodeUnfrozenEvent)
+		case e.RegistryRuntime != nil:
+			runtimeEvents = append(runtimeEvents, *e.RegistryRuntime)
+		case e.RegistryEntity != nil:
+			entityEvents = append(entityEvents, *e.RegistryEntity)
+		case e.RegistryNode != nil:
+			nodeEvents = append(nodeEvents, *e.RegistryNode)
+		case e.RegistryNodeUnfrozen != nil:
+			nodeUnfrozenEvents = append(nodeUnfrozenEvents, *e.RegistryNodeUnfrozen)
 		}
 	}
 
@@ -191,32 +183,44 @@ func (cc *ConsensusClient) StakingData(ctx context.Context, height int64) (*stor
 		return nil, err
 	}
 
-	var transfers []*stakingAPI.TransferEvent
-	var burns []*stakingAPI.BurnEvent
-	var escrows []*stakingAPI.EscrowEvent
-	var allowanceChanges []*stakingAPI.AllowanceChangeEvent
+	var transfers []nodeapi.TransferEvent
+	var burns []nodeapi.BurnEvent
+	var addEscrows []nodeapi.AddEscrowEvent
+	var reclaimEscrows []nodeapi.ReclaimEscrowEvent
+	var debondingStartEscrows []nodeapi.DebondingStartEscrowEvent
+	var takeEscrows []nodeapi.TakeEscrowEvent
+	var allowanceChanges []nodeapi.AllowanceChangeEvent
 
 	for _, event := range events {
 		switch e := event; {
-		case e.Transfer != nil:
-			transfers = append(transfers, event.Transfer)
-		case e.Burn != nil:
-			burns = append(burns, event.Burn)
-		case e.Escrow != nil:
-			escrows = append(escrows, event.Escrow)
-		case e.AllowanceChange != nil:
-			allowanceChanges = append(allowanceChanges, event.AllowanceChange)
+		case e.StakingTransfer != nil:
+			transfers = append(transfers, *event.StakingTransfer)
+		case e.StakingBurn != nil:
+			burns = append(burns, *event.StakingBurn)
+		case e.StakingAddEscrow != nil:
+			addEscrows = append(addEscrows, *event.StakingAddEscrow)
+		case e.StakingReclaimEscrow != nil:
+			reclaimEscrows = append(reclaimEscrows, *event.StakingReclaimEscrow)
+		case e.StakingDebondingStart != nil:
+			debondingStartEscrows = append(debondingStartEscrows, *event.StakingDebondingStart)
+		case e.StakingTakeEscrow != nil:
+			takeEscrows = append(takeEscrows, *event.StakingTakeEscrow)
+		case e.StakingAllowanceChange != nil:
+			allowanceChanges = append(allowanceChanges, *event.StakingAllowanceChange)
 		}
 	}
 
 	return &storage.StakingData{
-		Height:           height,
-		Epoch:            epoch,
-		Events:           events,
-		Transfers:        transfers,
-		Burns:            burns,
-		Escrows:          escrows,
-		AllowanceChanges: allowanceChanges,
+		Height:                height,
+		Epoch:                 epoch,
+		Events:                events,
+		Transfers:             transfers,
+		Burns:                 burns,
+		AddEscrows:            addEscrows,
+		ReclaimEscrows:        reclaimEscrows,
+		DebondingStartEscrows: debondingStartEscrows,
+		TakeEscrows:           takeEscrows,
+		AllowanceChanges:      allowanceChanges,
 	}, nil
 }
 
@@ -227,7 +231,7 @@ func (cc *ConsensusClient) SchedulerData(ctx context.Context, height int64) (*st
 		return nil, err
 	}
 
-	committees := make(map[common.Namespace][]*schedulerAPI.Committee, len(cc.network.ParaTimes.All))
+	committees := make(map[common.Namespace][]nodeapi.Committee, len(cc.network.ParaTimes.All))
 
 	for name := range cc.network.ParaTimes.All {
 		var runtimeID common.Namespace
@@ -256,29 +260,29 @@ func (cc *ConsensusClient) GovernanceData(ctx context.Context, height int64) (*s
 		return nil, err
 	}
 
-	var submissions []*governanceAPI.Proposal
-	var executions []*governanceAPI.ProposalExecutedEvent
-	var finalizations []*governanceAPI.Proposal
-	var votes []*governanceAPI.VoteEvent
+	var submissions []nodeapi.Proposal
+	var executions []nodeapi.ProposalExecutedEvent
+	var finalizations []nodeapi.Proposal
+	var votes []nodeapi.VoteEvent
 
 	for _, event := range events {
-		switch e := event; {
-		case e.ProposalSubmitted != nil:
-			proposal, err := cc.nodeApi.GetProposal(ctx, height, event.ProposalSubmitted.ID)
+		switch {
+		case event.GovernanceProposalSubmitted != nil:
+			proposal, err := cc.nodeApi.GetProposal(ctx, height, event.GovernanceProposalSubmitted.ID)
 			if err != nil {
 				return nil, err
 			}
-			submissions = append(submissions, proposal)
-		case e.ProposalExecuted != nil:
-			executions = append(executions, event.ProposalExecuted)
-		case e.ProposalFinalized != nil:
-			proposal, err := cc.nodeApi.GetProposal(ctx, height, event.ProposalFinalized.ID)
+			submissions = append(submissions, *proposal)
+		case event.GovernanceProposalExecuted != nil:
+			executions = append(executions, *event.GovernanceProposalExecuted)
+		case event.GovernanceProposalFinalized != nil:
+			proposal, err := cc.nodeApi.GetProposal(ctx, height, event.GovernanceProposalFinalized.ID)
 			if err != nil {
 				return nil, err
 			}
-			finalizations = append(finalizations, proposal)
-		case e.Vote != nil:
-			votes = append(votes, event.Vote)
+			finalizations = append(finalizations, *proposal)
+		case event.GovernanceVote != nil:
+			votes = append(votes, *event.GovernanceVote)
 		}
 	}
 	return &storage.GovernanceData{