Redesign validator set management to enable tracking all subnets (#1857)

Signed-off-by: Ceyhun Onur <[email protected]> Co-authored-by: Alberto Benegiamo <[email protected]> Co-authored-by: Stephen Buttolph <[email protected]>
ava-labs · Oct 23, 2023 · 7b7931b · 7b7931b
1 parent 4cd7051
commit 7b7931b
Show file tree

Hide file tree

Showing 76 changed files with 1,950 additions and 1,374 deletions.
diff --git a/api/info/service.go b/api/info/service.go
@@ -17,7 +17,6 @@ import (
 	"github.com/ava-labs/avalanchego/network"
 	"github.com/ava-labs/avalanchego/network/peer"
 	"github.com/ava-labs/avalanchego/snow/networking/benchlist"
-	"github.com/ava-labs/avalanchego/snow/validators"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/ips"
 	"github.com/ava-labs/avalanchego/utils/json"
@@ -37,7 +36,6 @@ type Info struct {
 	networking   network.Network
 	chainManager chains.Manager
 	vmManager    vms.Manager
-	validators   validators.Set
 	benchlist    benchlist.Manager
 }
 
@@ -65,7 +63,6 @@ func NewService(
 	vmManager vms.Manager,
 	myIP ips.DynamicIPPort,
 	network network.Network,
-	validators validators.Set,
 	benchlist benchlist.Manager,
 ) (http.Handler, error) {
 	server := rpc.NewServer()
@@ -80,7 +77,6 @@ func NewService(
 			vmManager:    vmManager,
 			myIP:         myIP,
 			networking:   network,
-			validators:   validators,
 			benchlist:    benchlist,
 		},
 		"info",

diff --git a/chains/manager.go b/chains/manager.go
@@ -148,15 +148,15 @@ type ChainParameters struct {
 	// The IDs of the feature extensions this chain is running.
 	FxIDs []ids.ID
 	// Invariant: Only used when [ID] is the P-chain ID.
-	CustomBeacons validators.Set
+	CustomBeacons validators.Manager
 }
 
 type chain struct {
 	Name    string
 	Context *snow.ConsensusContext
 	VM      common.VM
 	Handler handler.Handler
-	Beacons validators.Set
+	Beacons validators.Manager
 }
 
 // ChainConfig is configuration settings for the current execution.
@@ -531,15 +531,10 @@ func (m *manager) buildChain(chainParams ChainParameters, sb subnets.Subnet) (*c
 		}
 	}
 
-	var vdrs validators.Set // Validators validating this blockchain
-	var hasValidators bool
-	if m.SybilProtectionEnabled {
-		vdrs, hasValidators = m.Validators.Get(chainParams.SubnetID)
-	} else { // Sybil protection is disabled. Every peer validates every subnet.
-		vdrs, hasValidators = m.Validators.Get(constants.PrimaryNetworkID)
-	}
-	if !hasValidators {
-		return nil, fmt.Errorf("couldn't get validator set of subnet with ID %s. The subnet may not exist", chainParams.SubnetID)
+	vdrs := m.Validators
+	if !m.SybilProtectionEnabled {
+		// If sybil protection is disabled, everyone validates every chain
+		vdrs = validators.NewOverriddenManager(constants.PrimaryNetworkID, vdrs)
 	}
 
 	var chain *chain
@@ -594,7 +589,7 @@ func (m *manager) AddRegistrant(r Registrant) {
 func (m *manager) createAvalancheChain(
 	ctx *snow.ConsensusContext,
 	genesisData []byte,
-	vdrs validators.Set,
+	vdrs validators.Manager,
 	vm vertex.LinearizableVMWithEngine,
 	fxs []*common.Fx,
 	sb subnets.Subnet,
@@ -816,7 +811,10 @@ func (m *manager) createAvalancheChain(
 		appSender:    snowmanMessageSender,
 	}
 
-	bootstrapWeight := vdrs.Weight()
+	bootstrapWeight, err := vdrs.TotalWeight(ctx.SubnetID)
+	if err != nil {
+		return nil, fmt.Errorf("error while fetching weight for subnet %s: %w", ctx.SubnetID, err)
+	}
 
 	consensusParams := sb.Config().ConsensusParameters
 	sampleK := consensusParams.K
@@ -828,7 +826,7 @@ func (m *manager) createAvalancheChain(
 	if err != nil {
 		return nil, fmt.Errorf("error creating peer tracker: %w", err)
 	}
-	vdrs.RegisterCallbackListener(connectedValidators)
+	vdrs.RegisterCallbackListener(ctx.SubnetID, connectedValidators)
 
 	// Asynchronously passes messages from the network to the consensus engine
 	h, err := handler.New(
@@ -848,7 +846,7 @@ func (m *manager) createAvalancheChain(
 
 	connectedBeacons := tracker.NewPeers()
 	startupTracker := tracker.NewStartup(connectedBeacons, (3*bootstrapWeight+3)/4)
-	vdrs.RegisterCallbackListener(startupTracker)
+	vdrs.RegisterCallbackListener(ctx.SubnetID, startupTracker)
 
 	snowmanCommonCfg := common.Config{
 		Ctx:                            ctx,
@@ -998,8 +996,8 @@ func (m *manager) createAvalancheChain(
 func (m *manager) createSnowmanChain(
 	ctx *snow.ConsensusContext,
 	genesisData []byte,
-	vdrs validators.Set,
-	beacons validators.Set,
+	vdrs validators.Manager,
+	beacons validators.Manager,
 	vm block.ChainVM,
 	fxs []*common.Fx,
 	sb subnets.Subnet,
@@ -1164,7 +1162,10 @@ func (m *manager) createSnowmanChain(
 		return nil, err
 	}
 
-	bootstrapWeight := beacons.Weight()
+	bootstrapWeight, err := beacons.TotalWeight(ctx.SubnetID)
+	if err != nil {
+		return nil, fmt.Errorf("error while fetching weight for subnet %s: %w", ctx.SubnetID, err)
+	}
 
 	consensusParams := sb.Config().ConsensusParameters
 	sampleK := consensusParams.K
@@ -1176,7 +1177,7 @@ func (m *manager) createSnowmanChain(
 	if err != nil {
 		return nil, fmt.Errorf("error creating peer tracker: %w", err)
 	}
-	vdrs.RegisterCallbackListener(connectedValidators)
+	vdrs.RegisterCallbackListener(ctx.SubnetID, connectedValidators)
 
 	// Asynchronously passes messages from the network to the consensus engine
 	h, err := handler.New(
@@ -1196,7 +1197,7 @@ func (m *manager) createSnowmanChain(
 
 	connectedBeacons := tracker.NewPeers()
 	startupTracker := tracker.NewStartup(connectedBeacons, (3*bootstrapWeight+3)/4)
-	beacons.RegisterCallbackListener(startupTracker)
+	beacons.RegisterCallbackListener(ctx.SubnetID, startupTracker)
 
 	commonCfg := common.Config{
 		Ctx:                            ctx,
@@ -1358,7 +1359,7 @@ func (m *manager) registerBootstrappedHealthChecks() error {
 		if !m.IsBootstrapped(constants.PlatformChainID) {
 			return "node is currently bootstrapping", nil
 		}
-		if !validators.Contains(m.Validators, constants.PrimaryNetworkID, m.NodeID) {
+		if !m.Validators.Contains(constants.PrimaryNetworkID, m.NodeID) {
 			return "node is not a primary network validator", nil
 		}
 

diff --git a/network/config.go b/network/config.go
@@ -134,8 +134,8 @@ type Config struct {
 	TLSKey crypto.Signer `json:"-"`
 
 	// TrackedSubnets of the node.
-	TrackedSubnets set.Set[ids.ID] `json:"-"`
-	Beacons        validators.Set  `json:"-"`
+	TrackedSubnets set.Set[ids.ID]    `json:"-"`
+	Beacons        validators.Manager `json:"-"`
 
 	// Validators are the current validators in the Avalanche network
 	Validators validators.Manager `json:"-"`

diff --git a/network/example_test.go b/network/example_test.go
@@ -58,11 +58,11 @@ func (t *testExternalHandler) Disconnected(nodeID ids.NodeID) {
 	)
 }
 
-type testAggressiveValidatorSet struct {
-	validators.Set
+type testAggressiveValidatorManager struct {
+	validators.Manager
 }
 
-func (*testAggressiveValidatorSet) Contains(ids.NodeID) bool {
+func (*testAggressiveValidatorManager) Contains(ids.ID, ids.NodeID) bool {
 	return true
 }
 
@@ -78,8 +78,8 @@ func ExampleNewTestNetwork() {
 
 	// Needs to be periodically updated by the caller to have the latest
 	// validator set
-	validators := &testAggressiveValidatorSet{
-		Set: validators.NewSet(),
+	validators := &testAggressiveValidatorManager{
+		Manager: validators.NewManager(),
 	}
 
 	// If we want to be able to communicate with non-primary network subnets, we

diff --git a/network/network.go b/network/network.go
@@ -31,7 +31,6 @@ import (
 	"github.com/ava-labs/avalanchego/proto/pb/p2p"
 	"github.com/ava-labs/avalanchego/snow/networking/router"
 	"github.com/ava-labs/avalanchego/snow/networking/sender"
-	"github.com/ava-labs/avalanchego/snow/validators"
 	"github.com/ava-labs/avalanchego/subnets"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/ips"
@@ -55,12 +54,10 @@ var (
 	_ sender.ExternalSender = (*network)(nil)
 	_ Network               = (*network)(nil)
 
-	errMissingPrimaryValidators = errors.New("missing primary validator set")
-	errNotValidator             = errors.New("node is not a validator")
-	errNotTracked               = errors.New("subnet is not tracked")
-	errSubnetNotExist           = errors.New("subnet does not exist")
-	errExpectedProxy            = errors.New("expected proxy")
-	errExpectedTCPProtocol      = errors.New("expected TCP protocol")
+	errNotValidator        = errors.New("node is not a validator")
+	errNotTracked          = errors.New("subnet is not tracked")
+	errExpectedProxy       = errors.New("expected proxy")
+	errExpectedTCPProtocol = errors.New("expected TCP protocol")
 )
 
 // Network defines the functionality of the networking library.
@@ -189,11 +186,6 @@ func NewNetwork(
 	dialer dialer.Dialer,
 	router router.ExternalHandler,
 ) (Network, error) {
-	primaryNetworkValidators, ok := config.Validators.Get(constants.PrimaryNetworkID)
-	if !ok {
-		return nil, errMissingPrimaryValidators
-	}
-
 	if config.ProxyEnabled {
 		// Wrap the listener to process the proxy header.
 		listener = &proxyproto.Listener{
@@ -220,7 +212,7 @@ func NewNetwork(
 		log,
 		config.Namespace,
 		metricsRegisterer,
-		primaryNetworkValidators,
+		config.Validators,
 		config.ThrottlerConfig.InboundMsgThrottlerConfig,
 		config.ResourceTracker,
 		config.CPUTargeter,
@@ -234,7 +226,7 @@ func NewNetwork(
 		log,
 		config.Namespace,
 		metricsRegisterer,
-		primaryNetworkValidators,
+		config.Validators,
 		config.ThrottlerConfig.OutboundMsgThrottlerConfig,
 	)
 	if err != nil {
@@ -468,7 +460,7 @@ func (n *network) Connected(nodeID ids.NodeID) {
 // peer is a validator/beacon.
 func (n *network) AllowConnection(nodeID ids.NodeID) bool {
 	return !n.config.RequireValidatorToConnect ||
-		validators.Contains(n.config.Validators, constants.PrimaryNetworkID, n.config.MyNodeID) ||
+		n.config.Validators.Contains(constants.PrimaryNetworkID, n.config.MyNodeID) ||
 		n.WantsConnection(nodeID)
 }
 
@@ -807,7 +799,7 @@ func (n *network) WantsConnection(nodeID ids.NodeID) bool {
 }
 
 func (n *network) wantsConnection(nodeID ids.NodeID) bool {
-	return validators.Contains(n.config.Validators, constants.PrimaryNetworkID, nodeID) ||
+	return n.config.Validators.Contains(constants.PrimaryNetworkID, nodeID) ||
 		n.manuallyTrackedIDs.Contains(nodeID)
 }
 
@@ -862,7 +854,7 @@ func (n *network) getPeers(
 			continue
 		}
 
-		isValidator := validators.Contains(n.config.Validators, subnetID, nodeID)
+		isValidator := n.config.Validators.Contains(subnetID, nodeID)
 		// check if the peer is allowed to connect to the subnet
 		if !allower.IsAllowed(nodeID, isValidator) {
 			continue
@@ -881,14 +873,9 @@ func (n *network) samplePeers(
 	numPeersToSample int,
 	allower subnets.Allower,
 ) []peer.Peer {
-	subnetValidators, ok := n.config.Validators.Get(subnetID)
-	if !ok {
-		return nil
-	}
-
 	// If there are fewer validators than [numValidatorsToSample], then only
 	// sample [numValidatorsToSample] validators.
-	subnetValidatorsLen := subnetValidators.Len()
+	subnetValidatorsLen := n.config.Validators.Count(subnetID)
 	if subnetValidatorsLen < numValidatorsToSample {
 		numValidatorsToSample = subnetValidatorsLen
 	}
@@ -906,7 +893,7 @@ func (n *network) samplePeers(
 			}
 
 			peerID := p.ID()
-			isValidator := subnetValidators.Contains(peerID)
+			isValidator := n.config.Validators.Contains(subnetID, peerID)
 			// check if the peer is allowed to connect to the subnet
 			if !allower.IsAllowed(peerID, isValidator) {
 				return false
@@ -1343,18 +1330,18 @@ func (n *network) NodeUptime(subnetID ids.ID) (UptimeResult, error) {
 		return UptimeResult{}, errNotTracked
 	}
 
-	validators, ok := n.config.Validators.Get(subnetID)
-	if !ok {
-		return UptimeResult{}, errSubnetNotExist
-	}
-
-	myStake := validators.GetWeight(n.config.MyNodeID)
+	myStake := n.config.Validators.GetWeight(subnetID, n.config.MyNodeID)
 	if myStake == 0 {
 		return UptimeResult{}, errNotValidator
 	}
 
+	totalWeightInt, err := n.config.Validators.TotalWeight(subnetID)
+	if err != nil {
+		return UptimeResult{}, fmt.Errorf("error while fetching weight for subnet %s: %w", subnetID, err)
+	}
+
 	var (
-		totalWeight          = float64(validators.Weight())
+		totalWeight          = float64(totalWeightInt)
 		totalWeightedPercent = 100 * float64(myStake)
 		rewardingStake       = float64(myStake)
 	)
@@ -1366,7 +1353,7 @@ func (n *network) NodeUptime(subnetID ids.ID) (UptimeResult, error) {
 		peer, _ := n.connectedPeers.GetByIndex(i)
 
 		nodeID := peer.ID()
-		weight := validators.GetWeight(nodeID)
+		weight := n.config.Validators.GetWeight(subnetID, nodeID)
 		if weight == 0 {
 			// this is not a validator skip it.
 			continue