diff --git a/message/mock_outbound_message_builder.go b/message/mock_outbound_message_builder.go index 917d764028fe..1422230961e4 100644 --- a/message/mock_outbound_message_builder.go +++ b/message/mock_outbound_message_builder.go @@ -254,18 +254,18 @@ func (mr *MockOutboundMsgBuilderMockRecorder) GetAncestors(arg0, arg1, arg2, arg } // GetPeerList mocks base method. -func (m *MockOutboundMsgBuilder) GetPeerList(arg0, arg1 []byte) (OutboundMessage, error) { +func (m *MockOutboundMsgBuilder) GetPeerList(arg0, arg1 []byte, arg2 bool) (OutboundMessage, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPeerList", arg0, arg1) + ret := m.ctrl.Call(m, "GetPeerList", arg0, arg1, arg2) ret0, _ := ret[0].(OutboundMessage) ret1, _ := ret[1].(error) return ret0, ret1 } // GetPeerList indicates an expected call of GetPeerList. -func (mr *MockOutboundMsgBuilderMockRecorder) GetPeerList(arg0, arg1 any) *gomock.Call { +func (mr *MockOutboundMsgBuilderMockRecorder) GetPeerList(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPeerList", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).GetPeerList), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPeerList", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).GetPeerList), arg0, arg1, arg2) } // GetStateSummaryFrontier mocks base method. @@ -284,18 +284,18 @@ func (mr *MockOutboundMsgBuilderMockRecorder) GetStateSummaryFrontier(arg0, arg1 } // Handshake mocks base method. -func (m *MockOutboundMsgBuilder) Handshake(arg0 uint32, arg1 uint64, arg2 netip.AddrPort, arg3 string, arg4, arg5, arg6 uint32, arg7 uint64, arg8, arg9 []byte, arg10 []ids.ID, arg11, arg12 []uint32, arg13, arg14 []byte) (OutboundMessage, error) { +func (m *MockOutboundMsgBuilder) Handshake(arg0 uint32, arg1 uint64, arg2 netip.AddrPort, arg3 string, arg4, arg5, arg6 uint32, arg7 uint64, arg8, arg9 []byte, arg10 []ids.ID, arg11, arg12 []uint32, arg13, arg14 []byte, arg15 bool) (OutboundMessage, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Handshake", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14) + ret := m.ctrl.Call(m, "Handshake", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14, arg15) ret0, _ := ret[0].(OutboundMessage) ret1, _ := ret[1].(error) return ret0, ret1 } // Handshake indicates an expected call of Handshake. -func (mr *MockOutboundMsgBuilderMockRecorder) Handshake(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14 any) *gomock.Call { +func (mr *MockOutboundMsgBuilderMockRecorder) Handshake(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14, arg15 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handshake", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).Handshake), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handshake", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).Handshake), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14, arg15) } // PeerList mocks base method. diff --git a/message/outbound_msg_builder.go b/message/outbound_msg_builder.go index 78aacdce3e08..296e77446153 100644 --- a/message/outbound_msg_builder.go +++ b/message/outbound_msg_builder.go @@ -35,11 +35,13 @@ type OutboundMsgBuilder interface { objectedACPs []uint32, knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) GetPeerList( knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) PeerList( @@ -242,6 +244,7 @@ func (b *outMsgBuilder) Handshake( objectedACPs []uint32, knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) { subnetIDBytes := make([][]byte, len(trackedSubnets)) encodeIDs(trackedSubnets, subnetIDBytes) @@ -270,7 +273,8 @@ func (b *outMsgBuilder) Handshake( Filter: knownPeersFilter, Salt: knownPeersSalt, }, - IpBlsSig: ipBLSSig, + IpBlsSig: ipBLSSig, + AllSubnets: requestAllSubnetIPs, }, }, }, @@ -282,6 +286,7 @@ func (b *outMsgBuilder) Handshake( func (b *outMsgBuilder) GetPeerList( knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) { return b.builder.createOutbound( &p2p.Message{ @@ -291,6 +296,7 @@ func (b *outMsgBuilder) GetPeerList( Filter: knownPeersFilter, Salt: knownPeersSalt, }, + AllSubnets: requestAllSubnetIPs, }, }, }, diff --git a/network/README.md b/network/README.md index b4a6a01d648b..3d3341050e60 100644 --- a/network/README.md +++ b/network/README.md @@ -16,6 +16,7 @@ - [Bloom Filter](#bloom-filter) - [GetPeerList](#getpeerlist) - [PeerList](#peerlist) + - [Avoiding Persistent Network Traffic](#avoiding-persistent-network-traffic) ## Overview @@ -33,15 +34,15 @@ Peers communicate by enqueuing messages between one another. Each peer on either ```mermaid sequenceDiagram - actor Alice - actor Bob + actor Morty + actor Rick loop - Alice->>Bob: Write outbound messages - Bob->>Alice: Read incoming messages + Morty->>Rick: Write outbound messages + Rick->>Morty: Read incoming messages end loop - Bob->>Alice: Write outbound messages - Alice->>Bob: Read incoming messages + Rick->>Morty: Write outbound messages + Morty->>Rick: Read incoming messages end ``` @@ -55,50 +56,50 @@ A peer will then read the full message and attempt to parse it into either a net Upon connection to a new peer, a handshake is performed between the node attempting to establish the outbound connection to the peer and the peer receiving the inbound connection. -When attempting to establish the connection, the first message that the node sends is a `Handshake` message describing the compatibility of the nodes. If the `Handshake` message is successfully received and the peer decides that it wants a connection with this node, it replies with a `PeerList` message that contains metadata about other peers that allows a node to connect to them. See [Peerlist Gossip](#peerlist-gossip). +When attempting to establish the connection, the first message that the node sends is a `Handshake` message describing the configuration of the node. If the `Handshake` message is successfully received and the peer decides that it will allow a connection with this node, it replies with a `PeerList` message that contains metadata about other peers that allows a node to connect to them. See [PeerList Gossip](#peerlist-gossip). As an example, nodes that are attempting to connect with an incompatible version of AvalancheGo or a significantly skewed local clock are rejected. ```mermaid sequenceDiagram - actor Alice - actor Bob - Note over Alice,Bob: Connection Created + actor Morty + actor Rick + Note over Morty,Rick: Connection Created par - Alice->>Bob: AvalancheGo v1.0.0 + Morty->>Rick: AvalancheGo v1.0.0 and - Bob->>Alice: AvalancheGo v1.11.4 + Rick->>Morty: AvalancheGo v1.11.4 end - Note right of Bob: v1.0.0 is incompatible with v1.11.4. - Note left of Alice: v1.11.4 could be compatible with v1.0.0! + Note right of Rick: v1.0.0 is incompatible with v1.11.4. + Note left of Morty: v1.11.4 could be compatible with v1.0.0! par - Bob-->>Alice: Disconnect + Rick-->>Morty: Disconnect and - Alice-XBob: Peerlist + Morty-XRick: Peerlist end - Note over Alice,Bob: Handshake Failed + Note over Morty,Rick: Handshake Failed ``` Nodes that mutually desire the connection will both respond with `PeerList` messages and complete the handshake. ```mermaid sequenceDiagram - actor Alice - actor Bob - Note over Alice,Bob: Connection Created + actor Morty + actor Rick + Note over Morty,Rick: Connection Created par - Alice->>Bob: AvalancheGo v1.11.0 + Morty->>Rick: AvalancheGo v1.11.0 and - Bob->>Alice: AvalancheGo v1.11.4 + Rick->>Morty: AvalancheGo v1.11.4 end - Note right of Bob: v1.11.0 is compatible with v1.11.4! - Note left of Alice: v1.11.4 could be compatible with v1.11.0! + Note right of Rick: v1.11.0 is compatible with v1.11.4! + Note left of Morty: v1.11.4 could be compatible with v1.11.0! par - Bob->>Alice: Peerlist + Rick->>Morty: Peerlist and - Alice->>Bob: Peerlist + Morty->>Rick: Peerlist end - Note over Alice,Bob: Handshake Complete + Note over Morty,Rick: Handshake Complete ``` ### Ping-Pong Messages @@ -107,12 +108,12 @@ Peers periodically send `Ping` messages containing perceived uptime information. ```mermaid sequenceDiagram - actor Alice - actor Bob - Note left of Alice: Send Ping - Alice->>Bob: I think your uptime is 95% - Note right of Bob: Send Pong - Bob->>Alice: ACK + actor Morty + actor Rick + Note left of Morty: Send Ping + Morty->>Rick: I think your uptime is 95% + Note right of Rick: Send Pong + Rick->>Morty: ACK ``` ## Peer Discovery @@ -127,10 +128,12 @@ It is expected for Avalanche nodes to allow inbound connections. If a validator Avalanche nodes that have identified the `IP:Port` pair of a node they want to connect to will initiate outbound connections to this `IP:Port` pair. If the connection is not able to complete the [Peer Handshake](#peer-handshake), the connection will be re-attempted with an [Exponential Backoff](https://en.wikipedia.org/wiki/Exponential_backoff). -A node should initiate outbound connections to an `IP:Port` pair that is believed to belong to a node that is not connected and meets at least one of the following conditions: -- The node is in the initial bootstrapper set. -- The node is in the default bootstrapper set. -- The node in the current Primary Network validator set. +A node should initiate outbound connections to an `IP:Port` pair that is believed to belong to another node that is not connected and meets at least one of the following conditions: +- The peer is in the initial bootstrapper set. +- The peer is in the default bootstrapper set. +- The peer is a Primary Network validator. +- The peer is a validator of a tracked Subnet. +- The peer is a validator of a Subnet and the local node is a Primary Network validator. #### IP Authentication @@ -148,7 +151,16 @@ Once connected to an initial set of peers, a node can use these connections to d Peers are discovered by receiving [`PeerList`](#peerlist) messages during the [Peer Handshake](#peer-handshake). These messages quickly provide a node with knowledge of peers in the network. However, they offer no guarantee that the node will connect to and maintain connections with every peer in the network. -To provide an eventual guarantee that all peers learn of one another, nodes periodically send a [`GetPeerList`](#getpeerlist) message to a randomly selected validator with the node's current [Bloom Filter](#bloom-filter) and `Salt`. +To provide an eventual guarantee that all peers learn of one another, nodes periodically send a [`GetPeerList`](#getpeerlist) message to a randomly selected Primary Network validator with the node's current [Bloom Filter](#bloom-filter) and `Salt`. + +#### Gossipable Peers + +The peers that a node may include into a [`GetPeerList`](#getpeerlist) message are considered `gossipable`. + + +#### Trackable Peers + +The peers that a node would attempt to connect to if included in a [`PeerList`](#peerlist) message are considered `trackable`. #### Bloom Filter @@ -171,32 +183,42 @@ A `GetPeerList` message contains the Bloom Filter of the currently known peers a `PeerList` messages are expected to contain `IP:Port` pairs that satisfy all of the following constraints: - The Bloom Filter sent when requesting the `PeerList` message does not contain the node claiming the `IP:Port` pair. - The node claiming the `IP:Port` pair is currently connected. -- The `IP:Port` pair the node shared during the `Handshake` message is the node's most recently known `IP:Port` pair. -- The node claiming the `IP:Port` pair is either in the default bootstrapper set or is a current Primary Network validator. +- The node claiming the `IP:Port` pair is either in the default bootstrapper set, is a current Primary Network validator, is a validator of a tracked Subnet, or is a validator of a Subnet and the peer is a Primary Network validator. -#### Example PeerList Gossip +#### Avoiding Persistent Network Traffic -The following diagram shows an example of `Alice` repeatedly learning about new peers from `Bob`. +To avoid persistent network traffic, it must eventually hold that the set of [`gossipable peers`](#gossipable-peers) is a subset of the [`trackable peers`](#trackable-peers) for all nodes in the network. + +For example, say there are 3 nodes: `Rick`, `Morty`, and `Summer`. + +First we consider the case that `Rick` and `Morty` consider `Summer` [`gossipable`](#gossipable-peers) and [`trackable`](#trackable-peers), respectively. +```mermaid +sequenceDiagram + actor Morty + actor Rick + Note left of Morty: Not currently tracking Summer + Morty->>Rick: GetPeerList + Note right of Rick: Summer isn't in the bloom filter + Rick->>Morty: PeerList - Contains Summer + Note left of Morty: Track Summer and add to bloom filter + Morty->>Rick: GetPeerList + Note right of Rick: Summer is in the bloom filter + Rick->>Morty: PeerList - Empty +``` +This case is ideal, as `Rick` only notifies `Morty` about `Summer` once, and never uses bandwidth for their connection again. +Now we consider the case that `Rick` considers `Summer` [`gossipable`](#gossipable-peers), but `Morty` does not consider `Summer` [`trackable`](#trackable-peers). ```mermaid sequenceDiagram - actor Alice - actor Bob - Note left of Alice: Initialize Bloom Filter - Note left of Alice: Bloom: [0, 0, 0] - Alice->>Bob: GetPeerList [0, 0, 0] - Note right of Bob: Any peers can be sent. - Bob->>Alice: PeerList - Peer-1 - Note left of Alice: Bloom: [1, 0, 0] - Alice->>Bob: GetPeerList [1, 0, 0] - Note right of Bob: Either Peer-2 or Peer-3 can be sent. - Bob->>Alice: PeerList - Peer-3 - Note left of Alice: Bloom: [1, 0, 1] - Alice->>Bob: GetPeerList [1, 0, 1] - Note right of Bob: Only Peer-2 can be sent. - Bob->>Alice: PeerList - Peer-2 - Note left of Alice: Bloom: [1, 1, 1] - Alice->>Bob: GetPeerList [1, 1, 1] - Note right of Bob: There are no more peers left to send! - Bob->>Alice: PeerList - Empty + actor Morty + actor Rick + Note left of Morty: Not currently tracking Summer + Morty->>Rick: GetPeerList + Note right of Rick: Summer isn't in the bloom filter + Rick->>Morty: PeerList - Contains Summer + Note left of Morty: Ignore Summer + Morty->>Rick: GetPeerList + Note right of Rick: Summer isn't in the bloom filter + Rick->>Morty: PeerList - Contains Summer ``` +This case is suboptimal, because `Rick` told `Morty` about `Summer` multiple times. If this case were to happen consistently, `Rick` may waste a significant amount of bandwidth trying to teach `Morty` about `Summer`. \ No newline at end of file diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 47cb032fc7a4..9fa87e1c01e4 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -14,6 +14,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/snow/validators" "github.com/ava-labs/avalanchego/utils/bloom" + "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/crypto/bls" "github.com/ava-labs/avalanchego/utils/ips" "github.com/ava-labs/avalanchego/utils/logging" @@ -37,9 +38,10 @@ const ( newTimestamp = 2 ) -var _ validators.SetCallbackListener = (*ipTracker)(nil) +var _ validators.ManagerCallbackListener = (*ipTracker)(nil) func newIPTracker( + trackedSubnets set.Set[ids.ID], log logging.Logger, registerer prometheus.Registerer, ) (*ipTracker, error) { @@ -48,24 +50,30 @@ func newIPTracker( return nil, err } tracker := &ipTracker{ - log: log, - numTrackedIPs: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "tracked_ips", - Help: "Number of IPs this node is willing to dial", + trackedSubnets: trackedSubnets, + log: log, + numTrackedPeers: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "tracked_peers", + Help: "number of peers this node is monitoring", }), numGossipableIPs: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "gossipable_ips", - Help: "Number of IPs this node is willing to gossip", + Help: "number of IPs this node considers able to be gossiped", }), - bloomMetrics: bloomMetrics, - mostRecentTrackedIPs: make(map[ids.NodeID]*ips.ClaimedIPPort), - bloomAdditions: make(map[ids.NodeID]int), - connected: make(map[ids.NodeID]*ips.ClaimedIPPort), - gossipableIndices: make(map[ids.NodeID]int), + numTrackedSubnets: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "tracked_subnets", + Help: "number of subnets this node is monitoring", + }), + bloomMetrics: bloomMetrics, + tracked: make(map[ids.NodeID]*trackedNode), + bloomAdditions: make(map[ids.NodeID]int), + connected: make(map[ids.NodeID]*connectedNode), + subnet: make(map[ids.ID]*gossipableSubnet), } err = errors.Join( - registerer.Register(tracker.numTrackedIPs), + registerer.Register(tracker.numTrackedPeers), registerer.Register(tracker.numGossipableIPs), + registerer.Register(tracker.numTrackedSubnets), ) if err != nil { return nil, err @@ -73,30 +81,139 @@ func newIPTracker( return tracker, tracker.resetBloom() } -type ipTracker struct { - log logging.Logger - numTrackedIPs prometheus.Gauge +// A node is tracked if any of the following conditions are met: +// - The node was manually tracked +// - The node is a validator on any subnet +type trackedNode struct { + // manuallyTracked tracks if this node's connection was manually requested. + manuallyTracked bool + // validatedSubnets contains all the subnets that this node is a validator + // of, including potentially the primary network. + validatedSubnets set.Set[ids.ID] + // subnets contains the subset of [subnets] that the local node also tracks, + // including potentially the primary network. + trackedSubnets set.Set[ids.ID] + // ip is the most recently known IP of this node. + ip *ips.ClaimedIPPort +} + +func (n *trackedNode) wantsConnection() bool { + return n.manuallyTracked || n.trackedSubnets.Len() > 0 +} + +func (n *trackedNode) canDelete() bool { + return !n.manuallyTracked && n.validatedSubnets.Len() == 0 +} + +type connectedNode struct { + // trackedSubnets contains all the subnets that this node is syncing, + // including the primary network. + trackedSubnets set.Set[ids.ID] + // ip this node claimed when connecting. The IP is not necessarily the same + // IP as in the tracked map. + ip *ips.ClaimedIPPort +} + +type gossipableSubnet struct { numGossipableIPs prometheus.Gauge - bloomMetrics *bloom.Metrics - lock sync.RWMutex - // manuallyTracked contains the nodeIDs of all nodes whose connection was - // manually requested. - manuallyTracked set.Set[ids.NodeID] // manuallyGossipable contains the nodeIDs of all nodes whose IP was - // manually configured to be gossiped. + // manually configured to be gossiped for this subnet. manuallyGossipable set.Set[ids.NodeID] - // mostRecentTrackedIPs tracks the most recent IP of each node whose - // connection is desired. - // - // An IP is tracked if one of the following conditions are met: - // - The node was manually tracked - // - The node was manually requested to be gossiped - // - The node is a validator - mostRecentTrackedIPs map[ids.NodeID]*ips.ClaimedIPPort - // trackedIDs contains the nodeIDs of all nodes whose connection is desired. - trackedIDs set.Set[ids.NodeID] + // gossipableIDs contains the nodeIDs of all nodes whose IP could be + // gossiped. This is a superset of manuallyGossipable. + gossipableIDs set.Set[ids.NodeID] + + // An IP is marked as gossipable if all of the following conditions are met: + // - The node is a validator or was manually requested to be gossiped + // - The node is connected + // - The node reported that they are syncing this subnet + // - The IP the node connected with is its latest IP + gossipableIndices map[ids.NodeID]int + gossipableIPs []*ips.ClaimedIPPort +} + +func (s *gossipableSubnet) setGossipableIP(ip *ips.ClaimedIPPort) { + if index, ok := s.gossipableIndices[ip.NodeID]; ok { + s.gossipableIPs[index] = ip + return + } + + s.numGossipableIPs.Inc() + s.gossipableIndices[ip.NodeID] = len(s.gossipableIPs) + s.gossipableIPs = append(s.gossipableIPs, ip) +} + +func (s *gossipableSubnet) removeGossipableIP(nodeID ids.NodeID) { + indexToRemove, wasGossipable := s.gossipableIndices[nodeID] + if !wasGossipable { + return + } + + // If we aren't removing the last IP, we need to swap the last IP with the + // IP we are removing so that the slice is contiguous. + newNumGossipable := len(s.gossipableIPs) - 1 + if newNumGossipable != indexToRemove { + replacementIP := s.gossipableIPs[newNumGossipable] + s.gossipableIndices[replacementIP.NodeID] = indexToRemove + s.gossipableIPs[indexToRemove] = replacementIP + } + + s.numGossipableIPs.Dec() + delete(s.gossipableIndices, nodeID) + s.gossipableIPs[newNumGossipable] = nil + s.gossipableIPs = s.gossipableIPs[:newNumGossipable] +} + +// [maxNumIPs] applies to the total number of IPs returned, including the IPs +// initially provided in [ips]. +// [ips] and [nodeIDs] are extended and returned with the additional IPs added. +func (s *gossipableSubnet) getGossipableIPs( + exceptNodeID ids.NodeID, + exceptIPs *bloom.ReadFilter, + salt []byte, + maxNumIPs int, + ips []*ips.ClaimedIPPort, + nodeIDs set.Set[ids.NodeID], +) ([]*ips.ClaimedIPPort, set.Set[ids.NodeID]) { + uniform := sampler.NewUniform() + uniform.Initialize(uint64(len(s.gossipableIPs))) + + for len(ips) < maxNumIPs { + index, hasNext := uniform.Next() + if !hasNext { + return ips, nodeIDs + } + + ip := s.gossipableIPs[index] + if ip.NodeID == exceptNodeID || + nodeIDs.Contains(ip.NodeID) || + bloom.Contains(exceptIPs, ip.GossipID[:], salt) { + continue + } + + ips = append(ips, ip) + nodeIDs.Add(ip.NodeID) + } + return ips, nodeIDs +} + +func (s *gossipableSubnet) canDelete() bool { + return s.gossipableIDs.Len() == 0 +} + +type ipTracker struct { + // trackedSubnets does not include the primary network. + trackedSubnets set.Set[ids.ID] + log logging.Logger + numTrackedPeers prometheus.Gauge + numGossipableIPs prometheus.Gauge // IPs are not deduplicated across subnets + numTrackedSubnets prometheus.Gauge + bloomMetrics *bloom.Metrics + + lock sync.RWMutex + tracked map[ids.NodeID]*trackedNode // The bloom filter contains the most recent tracked IPs to avoid // unnecessary IP gossip. @@ -108,19 +225,11 @@ type ipTracker struct { bloomSalt []byte maxBloomCount int - // Connected tracks the IP of currently connected peers, including tracked - // and untracked nodes. The IP is not necessarily the same IP as in - // mostRecentTrackedIPs. - connected map[ids.NodeID]*ips.ClaimedIPPort - - // An IP is marked as gossipable if all of the following conditions are met: - // - The node is a validator or was manually requested to be gossiped - // - The node is connected - // - The IP the node connected with is its latest IP - gossipableIndices map[ids.NodeID]int - // gossipableIPs is guaranteed to be a subset of [mostRecentTrackedIPs]. - gossipableIPs []*ips.ClaimedIPPort - gossipableIDs set.Set[ids.NodeID] + // Connected tracks the information of currently connected peers, including + // tracked and untracked nodes. + connected map[ids.NodeID]*connectedNode + // subnet tracks all the subnets that have at least one gossipable ID. + subnet map[ids.ID]*gossipableSubnet } // ManuallyTrack marks the provided nodeID as being desirable to connect to. @@ -134,8 +243,7 @@ func (i *ipTracker) ManuallyTrack(nodeID ids.NodeID) { i.lock.Lock() defer i.lock.Unlock() - i.addTrackableID(nodeID) - i.manuallyTracked.Add(nodeID) + i.addTrackableID(nodeID, nil) } // ManuallyGossip marks the provided nodeID as being desirable to connect to and @@ -143,107 +251,142 @@ func (i *ipTracker) ManuallyTrack(nodeID ids.NodeID) { // // In order to avoid persistent network gossip, it's important for nodes in the // network to agree upon manually gossiped nodeIDs. -func (i *ipTracker) ManuallyGossip(nodeID ids.NodeID) { +func (i *ipTracker) ManuallyGossip(subnetID ids.ID, nodeID ids.NodeID) { i.lock.Lock() defer i.lock.Unlock() - i.addTrackableID(nodeID) - i.manuallyTracked.Add(nodeID) + if subnetID == constants.PrimaryNetworkID || i.trackedSubnets.Contains(subnetID) { + i.addTrackableID(nodeID, nil) + } - i.addGossipableID(nodeID) - i.manuallyGossipable.Add(nodeID) + i.addTrackableID(nodeID, &subnetID) + i.addGossipableID(nodeID, subnetID, true) } // WantsConnection returns true if any of the following conditions are met: // 1. The node has been manually tracked. -// 2. The node has been manually gossiped. -// 3. The node is currently a validator. +// 2. The node has been manually gossiped on a tracked subnet. +// 3. The node is currently a validator on a tracked subnet. func (i *ipTracker) WantsConnection(nodeID ids.NodeID) bool { i.lock.RLock() defer i.lock.RUnlock() - return i.trackedIDs.Contains(nodeID) + node, ok := i.tracked[nodeID] + return ok && node.wantsConnection() } // ShouldVerifyIP is used as an optimization to avoid unnecessary IP // verification. It returns true if all of the following conditions are met: // 1. The provided IP is from a node whose connection is desired. // 2. This IP is newer than the most recent IP we know of for the node. -func (i *ipTracker) ShouldVerifyIP(ip *ips.ClaimedIPPort) bool { +func (i *ipTracker) ShouldVerifyIP( + ip *ips.ClaimedIPPort, + trackAllSubnets bool, +) bool { i.lock.RLock() defer i.lock.RUnlock() - if !i.trackedIDs.Contains(ip.NodeID) { + node, ok := i.tracked[ip.NodeID] + if !ok { + return false + } + + if !trackAllSubnets && !node.wantsConnection() { return false } - prevIP, ok := i.mostRecentTrackedIPs[ip.NodeID] - return !ok || // This would be the first IP - prevIP.Timestamp < ip.Timestamp // This would be a newer IP + return node.ip == nil || // This would be the first IP + node.ip.Timestamp < ip.Timestamp // This would be a newer IP } // AddIP attempts to update the node's IP to the provided IP. This function // assumes the provided IP has been verified. Returns true if all of the // following conditions are met: -// 1. The provided IP is from a node whose connection is desired. +// 1. The provided IP is from a node whose connection is desired on a tracked +// subnet. // 2. This IP is newer than the most recent IP we know of for the node. // -// If the previous IP was marked as gossipable, calling this function will -// remove the IP from the gossipable set. +// If this IP is replacing a gossipable IP, this IP will also be marked as +// gossipable. func (i *ipTracker) AddIP(ip *ips.ClaimedIPPort) bool { i.lock.Lock() defer i.lock.Unlock() - return i.addIP(ip) > sameTimestamp + timestampComparison, trackedNode := i.addIP(ip) + if timestampComparison <= sameTimestamp { + return false + } + + if connectedNode, ok := i.connected[ip.NodeID]; ok { + i.setGossipableIP(trackedNode.ip, connectedNode.trackedSubnets) + } + return trackedNode.wantsConnection() } -// GetIP returns the most recent IP of the provided nodeID. If a connection to -// this nodeID is not desired, this function will return false. +// GetIP returns the most recent IP of the provided nodeID. Returns true if all +// of the following conditions are met: +// 1. There is currently an IP for the provided nodeID. +// 2. The provided IP is from a node whose connection is desired on a tracked +// subnet. func (i *ipTracker) GetIP(nodeID ids.NodeID) (*ips.ClaimedIPPort, bool) { i.lock.RLock() defer i.lock.RUnlock() - ip, ok := i.mostRecentTrackedIPs[nodeID] - return ip, ok + node, ok := i.tracked[nodeID] + if !ok || node.ip == nil { + return nil, false + } + return node.ip, node.wantsConnection() } // Connected is called when a connection is established. The peer should have // provided [ip] during the handshake. -func (i *ipTracker) Connected(ip *ips.ClaimedIPPort) { +func (i *ipTracker) Connected(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids.ID]) { i.lock.Lock() defer i.lock.Unlock() - i.connected[ip.NodeID] = ip - if i.addIP(ip) >= sameTimestamp && i.gossipableIDs.Contains(ip.NodeID) { - i.addGossipableIP(ip) + i.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: trackedSubnets, + ip: ip, } -} -func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) int { - if !i.trackedIDs.Contains(ip.NodeID) { - return untrackedTimestamp + timestampComparison, trackedNode := i.addIP(ip) + if timestampComparison != untrackedTimestamp { + i.setGossipableIP(trackedNode.ip, trackedSubnets) } +} - prevIP, ok := i.mostRecentTrackedIPs[ip.NodeID] +func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, *trackedNode) { + node, ok := i.tracked[ip.NodeID] if !ok { + return untrackedTimestamp, nil + } + + if node.ip == nil { // This is the first IP we've heard from the validator, so it is the // most recent. - i.updateMostRecentTrackedIP(ip) - // Because we didn't previously have an IP, we know we aren't currently - // connected to them. - return newTimestamp + i.updateMostRecentTrackedIP(node, ip) + return newTimestamp, node } - if prevIP.Timestamp > ip.Timestamp { - return olderTimestamp // This IP is old than the previously known IP. + if node.ip.Timestamp > ip.Timestamp { + return olderTimestamp, node // This IP is older than the previously known IP. } - if prevIP.Timestamp == ip.Timestamp { - return sameTimestamp // This IP is equal to the previously known IP. + if node.ip.Timestamp == ip.Timestamp { + return sameTimestamp, node // This IP is equal to the previously known IP. } - i.updateMostRecentTrackedIP(ip) - i.removeGossipableIP(ip.NodeID) - return newerTimestamp + // This IP is newer than the previously known IP. + i.updateMostRecentTrackedIP(node, ip) + return newerTimestamp, node +} + +func (i *ipTracker) setGossipableIP(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids.ID]) { + for subnetID := range trackedSubnets { + if subnet, ok := i.subnet[subnetID]; ok && subnet.gossipableIDs.Contains(ip.NodeID) { + subnet.setGossipableIP(ip) + } + } } // Disconnected is called when a connection to the peer is closed. @@ -251,77 +394,135 @@ func (i *ipTracker) Disconnected(nodeID ids.NodeID) { i.lock.Lock() defer i.lock.Unlock() + connectedNode, ok := i.connected[nodeID] + if !ok { + return + } delete(i.connected, nodeID) - i.removeGossipableIP(nodeID) + + for subnetID := range connectedNode.trackedSubnets { + if subnet, ok := i.subnet[subnetID]; ok { + subnet.removeGossipableIP(nodeID) + } + } } -func (i *ipTracker) OnValidatorAdded(nodeID ids.NodeID, _ *bls.PublicKey, _ ids.ID, _ uint64) { +func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls.PublicKey, _ ids.ID, _ uint64) { i.lock.Lock() defer i.lock.Unlock() - i.addTrackableID(nodeID) - i.addGossipableID(nodeID) + i.addTrackableID(nodeID, &subnetID) + i.addGossipableID(nodeID, subnetID, false) } -func (i *ipTracker) addTrackableID(nodeID ids.NodeID) { - if i.trackedIDs.Contains(nodeID) { +// If [subnetID] is nil, the nodeID is being manually tracked. +func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { + nodeTracker, previouslyTracked := i.tracked[nodeID] + if !previouslyTracked { + i.numTrackedPeers.Inc() + nodeTracker = &trackedNode{} + i.tracked[nodeID] = nodeTracker + } + + if subnetID == nil { + nodeTracker.manuallyTracked = true + } else { + nodeTracker.validatedSubnets.Add(*subnetID) + if *subnetID == constants.PrimaryNetworkID || i.trackedSubnets.Contains(*subnetID) { + nodeTracker.trackedSubnets.Add(*subnetID) + } + } + + if previouslyTracked { return } - i.trackedIDs.Add(nodeID) - ip, connected := i.connected[nodeID] + node, connected := i.connected[nodeID] if !connected { return } // Because we previously weren't tracking this nodeID, the IP from the // connection is guaranteed to be the most up-to-date IP that we know. - i.updateMostRecentTrackedIP(ip) + i.updateMostRecentTrackedIP(nodeTracker, node.ip) } -func (i *ipTracker) addGossipableID(nodeID ids.NodeID) { - if i.gossipableIDs.Contains(nodeID) { - return +func (i *ipTracker) addGossipableID(nodeID ids.NodeID, subnetID ids.ID, manuallyGossiped bool) { + subnet, ok := i.subnet[subnetID] + if !ok { + i.numTrackedSubnets.Inc() + subnet = &gossipableSubnet{ + numGossipableIPs: i.numGossipableIPs, + gossipableIndices: make(map[ids.NodeID]int), + } + i.subnet[subnetID] = subnet } - i.gossipableIDs.Add(nodeID) - connectedIP, connected := i.connected[nodeID] - if !connected { + if manuallyGossiped { + subnet.manuallyGossipable.Add(nodeID) + } + if subnet.gossipableIDs.Contains(nodeID) { return } - if updatedIP, ok := i.mostRecentTrackedIPs[nodeID]; !ok || connectedIP.Timestamp != updatedIP.Timestamp { + subnet.gossipableIDs.Add(nodeID) + node, connected := i.connected[nodeID] + if !connected || !node.trackedSubnets.Contains(subnetID) { return } - i.addGossipableIP(connectedIP) + if trackedNode, ok := i.tracked[nodeID]; ok { + subnet.setGossipableIP(trackedNode.ip) + } } -func (*ipTracker) OnValidatorWeightChanged(ids.NodeID, uint64, uint64) {} +func (*ipTracker) OnValidatorWeightChanged(ids.ID, ids.NodeID, uint64, uint64) {} -func (i *ipTracker) OnValidatorRemoved(nodeID ids.NodeID, _ uint64) { +func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uint64) { i.lock.Lock() defer i.lock.Unlock() - if i.manuallyGossipable.Contains(nodeID) { + subnet, ok := i.subnet[subnetID] + if !ok { + i.log.Error("attempted removal of validator from untracked subnet", + zap.Stringer("subnetID", subnetID), + zap.Stringer("nodeID", nodeID), + ) + return + } + + if subnet.manuallyGossipable.Contains(nodeID) { return } - i.gossipableIDs.Remove(nodeID) - i.removeGossipableIP(nodeID) + subnet.gossipableIDs.Remove(nodeID) + subnet.removeGossipableIP(nodeID) - if i.manuallyTracked.Contains(nodeID) { + if subnet.canDelete() { + i.numTrackedSubnets.Dec() + delete(i.subnet, subnetID) + } + + trackedNode, ok := i.tracked[nodeID] + if !ok { + i.log.Error("attempted removal of untracked validator", + zap.Stringer("subnetID", subnetID), + zap.Stringer("nodeID", nodeID), + ) return } - i.trackedIDs.Remove(nodeID) - delete(i.mostRecentTrackedIPs, nodeID) - i.numTrackedIPs.Set(float64(len(i.mostRecentTrackedIPs))) + trackedNode.validatedSubnets.Remove(subnetID) + trackedNode.trackedSubnets.Remove(subnetID) + + if trackedNode.canDelete() { + i.numTrackedPeers.Dec() + delete(i.tracked, nodeID) + } } -func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { - i.mostRecentTrackedIPs[ip.NodeID] = ip - i.numTrackedIPs.Set(float64(len(i.mostRecentTrackedIPs))) +func (i *ipTracker) updateMostRecentTrackedIP(node *trackedNode, ip *ips.ClaimedIPPort) { + node.ip = ip oldCount := i.bloomAdditions[ip.NodeID] if oldCount >= maxIPEntriesPerNode { @@ -350,68 +551,6 @@ func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { i.bloomMetrics.Count.Inc() } -func (i *ipTracker) addGossipableIP(ip *ips.ClaimedIPPort) { - i.gossipableIndices[ip.NodeID] = len(i.gossipableIPs) - i.gossipableIPs = append(i.gossipableIPs, ip) - i.numGossipableIPs.Inc() -} - -func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { - indexToRemove, wasGossipable := i.gossipableIndices[nodeID] - if !wasGossipable { - return - } - - newNumGossipable := len(i.gossipableIPs) - 1 - if newNumGossipable != indexToRemove { - replacementIP := i.gossipableIPs[newNumGossipable] - i.gossipableIndices[replacementIP.NodeID] = indexToRemove - i.gossipableIPs[indexToRemove] = replacementIP - } - - delete(i.gossipableIndices, nodeID) - i.gossipableIPs[newNumGossipable] = nil - i.gossipableIPs = i.gossipableIPs[:newNumGossipable] - i.numGossipableIPs.Dec() -} - -// GetGossipableIPs returns the latest IPs of connected validators. The returned -// IPs will not contain [exceptNodeID] or any IPs contained in [exceptIPs]. If -// the number of eligible IPs to return low, it's possible that every IP will be -// iterated over while handling this call. -func (i *ipTracker) GetGossipableIPs( - exceptNodeID ids.NodeID, - exceptIPs *bloom.ReadFilter, - salt []byte, - maxNumIPs int, -) []*ips.ClaimedIPPort { - var ( - uniform = sampler.NewUniform() - ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) - ) - - i.lock.RLock() - defer i.lock.RUnlock() - - uniform.Initialize(uint64(len(i.gossipableIPs))) - for len(ips) < maxNumIPs { - index, hasNext := uniform.Next() - if !hasNext { - return ips - } - - ip := i.gossipableIPs[index] - if ip.NodeID == exceptNodeID { - continue - } - - if !bloom.Contains(exceptIPs, ip.GossipID[:], salt) { - ips = append(ips, ip) - } - } - return ips -} - // ResetBloom prunes the current bloom filter. This must be called periodically // to ensure that validators that change their IPs are updated correctly and // that validators that left the validator set are removed. @@ -441,7 +580,7 @@ func (i *ipTracker) resetBloom() error { return err } - count := max(maxIPEntriesPerNode*i.trackedIDs.Len(), minCountEstimate) + count := max(maxIPEntriesPerNode*len(i.tracked), minCountEstimate) numHashes, numEntries := bloom.OptimalParameters( count, targetFalsePositiveProbability, @@ -456,10 +595,56 @@ func (i *ipTracker) resetBloom() error { i.bloomSalt = newSalt i.maxBloomCount = bloom.EstimateCount(numHashes, numEntries, maxFalsePositiveProbability) - for nodeID, ip := range i.mostRecentTrackedIPs { - bloom.Add(newFilter, ip.GossipID[:], newSalt) + for nodeID, trackedNode := range i.tracked { + if trackedNode.ip == nil { + continue + } + + bloom.Add(newFilter, trackedNode.ip.GossipID[:], newSalt) i.bloomAdditions[nodeID] = 1 } i.bloomMetrics.Reset(newFilter, i.maxBloomCount) return nil } + +func getGossipableIPs[T any]( + i *ipTracker, + iter map[ids.ID]T, // The values in this map aren't actually used. + allowed func(ids.ID) bool, + exceptNodeID ids.NodeID, + exceptIPs *bloom.ReadFilter, + salt []byte, + maxNumIPs int, +) []*ips.ClaimedIPPort { + var ( + ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) + nodeIDs = set.NewSet[ids.NodeID](maxNumIPs) + ) + + i.lock.RLock() + defer i.lock.RUnlock() + + for subnetID := range iter { + if !allowed(subnetID) { + continue + } + + subnet, ok := i.subnet[subnetID] + if !ok { + continue + } + + ips, nodeIDs = subnet.getGossipableIPs( + exceptNodeID, + exceptIPs, + salt, + maxNumIPs, + ips, + nodeIDs, + ) + if len(ips) >= maxNumIPs { + break + } + } + return ips +} diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index bbfbdb958773..99f1a616287e 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -12,12 +12,18 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/bloom" + "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/ips" "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/set" ) func newTestIPTracker(t *testing.T) *ipTracker { - tracker, err := newIPTracker(logging.NoLog{}, prometheus.NewRegistry()) + tracker, err := newIPTracker( + nil, + logging.NoLog{}, + prometheus.NewRegistry(), + ) require.NoError(t, err) return tracker } @@ -33,800 +39,1135 @@ func newerTestIP(ip *ips.ClaimedIPPort) *ips.ClaimedIPPort { func requireEqual(t *testing.T, expected, actual *ipTracker) { require := require.New(t) - require.Equal(expected.manuallyTracked, actual.manuallyTracked) - require.Equal(expected.manuallyGossipable, actual.manuallyGossipable) - require.Equal(expected.mostRecentTrackedIPs, actual.mostRecentTrackedIPs) - require.Equal(expected.trackedIDs, actual.trackedIDs) + require.Equal(expected.tracked, actual.tracked) require.Equal(expected.bloomAdditions, actual.bloomAdditions) require.Equal(expected.maxBloomCount, actual.maxBloomCount) require.Equal(expected.connected, actual.connected) - require.Equal(expected.gossipableIndices, actual.gossipableIndices) - require.Equal(expected.gossipableIPs, actual.gossipableIPs) - require.Equal(expected.gossipableIDs, actual.gossipableIDs) + require.Equal(expected.subnet, actual.subnet) } func requireMetricsConsistent(t *testing.T, tracker *ipTracker) { require := require.New(t) - require.Equal(float64(len(tracker.mostRecentTrackedIPs)), testutil.ToFloat64(tracker.numTrackedIPs)) - require.Equal(float64(len(tracker.gossipableIPs)), testutil.ToFloat64(tracker.numGossipableIPs)) + require.Equal(float64(len(tracker.tracked)), testutil.ToFloat64(tracker.numTrackedPeers)) + var numGossipableIPs int + for _, subnet := range tracker.subnet { + numGossipableIPs += len(subnet.gossipableIndices) + } + require.Equal(float64(numGossipableIPs), testutil.ToFloat64(tracker.numGossipableIPs)) + require.Equal(float64(len(tracker.subnet)), testutil.ToFloat64(tracker.numTrackedSubnets)) require.Equal(float64(tracker.bloom.Count()), testutil.ToFloat64(tracker.bloomMetrics.Count)) require.Equal(float64(tracker.maxBloomCount), testutil.ToFloat64(tracker.bloomMetrics.MaxCount)) } func TestIPTracker_ManuallyTrack(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + expectedChange func(*ipTracker) }{ { name: "non-connected non-validator", - initialState: newTestIPTracker(t), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.trackedIDs.Add(ip.NodeID) - return tracker - }(), + initialState: newTestIPTracker, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + } + }, }, { name: "connected non-validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip - tracker.trackedIDs.Add(ip.NodeID) + }, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + ip: ip, + } tracker.bloomAdditions[ip.NodeID] = 1 - return tracker - }(), + }, }, { - name: "non-connected validator", - initialState: func() *ipTracker { + name: "non-connected tracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + }, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].manuallyTracked = true + }, + }, + { + name: "non-connected untracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].manuallyTracked = true + }, }, { - name: "connected validator", - initialState: func() *ipTracker { + name: "connected tracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + }, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].manuallyTracked = true + }, + }, + { + name: "connected untracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].manuallyTracked = true + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.ManuallyTrack(test.nodeID) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.ManuallyTrack(ip.NodeID) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } func TestIPTracker_ManuallyGossip(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + subnetID ids.ID + expectedChange func(*ipTracker) }{ { - name: "non-connected non-validator", - initialState: newTestIPTracker(t), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + name: "non-connected tracked non-validator", + initialState: newTestIPTracker, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + } + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } + }, + }, + { + name: "non-connected untracked non-validator", + initialState: newTestIPTracker, + subnetID: subnetID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + validatedSubnets: set.Of(subnetID), + } + tracker.subnet[subnetID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } + }, + }, + { + name: "connected tracked non-validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) - tracker.trackedIDs.Add(ip.NodeID) - tracker.gossipableIDs.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Inc() + tracker.numGossipableIPs.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + ip, + }, + } + }, }, { - name: "connected non-validator", - initialState: func() *ipTracker { + name: "connected untracked non-validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip - tracker.trackedIDs.Add(ip.NodeID) + }, + subnetID: subnetID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + validatedSubnets: set.Of(subnetID), + ip: ip, + } tracker.bloomAdditions[ip.NodeID] = 1 - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ - ip, + tracker.subnet[subnetID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), } - tracker.gossipableIDs.Add(ip.NodeID) - return tracker - }(), + }, }, { - name: "non-connected validator", - initialState: func() *ipTracker { + name: "non-connected tracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].manuallyTracked = true + tracker.subnet[constants.PrimaryNetworkID].manuallyGossipable = set.Of(ip.NodeID) + }, + }, + { + name: "non-connected untracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, + subnetID: subnetID, + expectedChange: func(tracker *ipTracker) { + tracker.subnet[subnetID].manuallyGossipable = set.Of(ip.NodeID) + }, }, { - name: "connected validator", - initialState: func() *ipTracker { + name: "connected tracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].manuallyTracked = true + tracker.subnet[constants.PrimaryNetworkID].manuallyGossipable = set.Of(ip.NodeID) + }, + }, + { + name: "connected untracked validator", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, + subnetID: subnetID, + expectedChange: func(tracker *ipTracker) { + tracker.subnet[subnetID].manuallyGossipable = set.Of(ip.NodeID) + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.ManuallyGossip(test.nodeID) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.ManuallyGossip(test.subnetID, ip.NodeID) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } -func TestIPTracker_AddIP(t *testing.T) { +func TestIPTracker_ShouldVerifyIP(t *testing.T) { newerIP := newerTestIP(ip) tests := []struct { - name string - initialState *ipTracker - ip *ips.ClaimedIPPort - expectedUpdated bool - expectedState *ipTracker + name string + tracker func(t *testing.T) *ipTracker + ip *ips.ClaimedIPPort + expectedTrackAllSubnets bool + expectedTrackRequestedSubnets bool }{ { - name: "non-validator", - initialState: newTestIPTracker(t), - ip: ip, - expectedUpdated: false, - expectedState: newTestIPTracker(t), + name: "node not tracked", + tracker: newTestIPTracker, + ip: ip, + expectedTrackAllSubnets: false, + expectedTrackRequestedSubnets: false, }, { - name: "first known IP", - initialState: func() *ipTracker { + name: "undesired connection", + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(ids.GenerateTestID(), ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - ip: ip, - expectedUpdated: true, - expectedState: func() *ipTracker { + }, + ip: ip, + expectedTrackAllSubnets: true, + expectedTrackRequestedSubnets: false, + }, + { + name: "desired connection first IP", + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip - tracker.bloomAdditions[ip.NodeID] = 1 + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, + ip: ip, + expectedTrackAllSubnets: true, + expectedTrackRequestedSubnets: true, }, { - name: "older IP", - initialState: func() *ipTracker { + name: "desired connection older IP", + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker - }(), - ip: ip, - expectedUpdated: false, - expectedState: func() *ipTracker { + }, + ip: ip, + expectedTrackAllSubnets: false, + expectedTrackRequestedSubnets: false, + }, + { + name: "desired connection same IP", + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(newerIP)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, + ip: ip, + expectedTrackAllSubnets: false, + expectedTrackRequestedSubnets: false, }, { - name: "same IP", - initialState: func() *ipTracker { + name: "desired connection newer IP", + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), - ip: ip, - expectedUpdated: false, - expectedState: func() *ipTracker { + }, + ip: newerIP, + expectedTrackAllSubnets: true, + expectedTrackRequestedSubnets: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + + tracker := test.tracker(t) + require.Equal(test.expectedTrackAllSubnets, tracker.ShouldVerifyIP(test.ip, true)) + require.Equal(test.expectedTrackRequestedSubnets, tracker.ShouldVerifyIP(test.ip, false)) + }) + } +} + +func TestIPTracker_AddIP(t *testing.T) { + subnetID := ids.GenerateTestID() + newerIP := newerTestIP(ip) + tests := []struct { + name string + initialState func(t *testing.T) *ipTracker + ip *ips.ClaimedIPPort + expectedChange func(*ipTracker) + expectedUpdatedAndDesired bool + }{ + { + name: "non-validator", + initialState: newTestIPTracker, + ip: ip, + expectedChange: func(*ipTracker) {}, + expectedUpdatedAndDesired: false, + }, + { + name: "first known IP of tracked node", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].ip = ip + tracker.bloomAdditions[ip.NodeID] = 1 + }, + expectedUpdatedAndDesired: true, + }, + { + name: "first known IP of untracked node", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }, + ip: ip, + expectedUpdatedAndDesired: false, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].ip = ip + tracker.bloomAdditions[ip.NodeID] = 1 + }, }, { - name: "disconnected newer IP", - initialState: func() *ipTracker { + name: "older IP of tracked node", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + require.True(t, tracker.AddIP(newerIP)) + return tracker + }, + ip: ip, + expectedUpdatedAndDesired: false, + expectedChange: func(*ipTracker) {}, + }, + { + name: "older IP of untracked node", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(newerIP)) + return tracker + }, + ip: ip, + expectedUpdatedAndDesired: false, + expectedChange: func(*ipTracker) {}, + }, + { + name: "same IP of tracked node", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), - ip: newerIP, - expectedUpdated: true, - expectedState: func() *ipTracker { + }, + ip: ip, + expectedUpdatedAndDesired: false, + expectedChange: func(*ipTracker) {}, + }, + { + name: "same IP of untracked node", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + return tracker + }, + ip: ip, + expectedUpdatedAndDesired: false, + expectedChange: func(*ipTracker) {}, + }, + { + name: "disconnected newer IP of tracked node", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) - tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP - tracker.bloomAdditions[newerIP.NodeID] = 2 return tracker - }(), + }, + ip: newerIP, + expectedUpdatedAndDesired: true, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[newerIP.NodeID].ip = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + }, }, { - name: "connected newer IP", - initialState: func() *ipTracker { + name: "disconnected newer IP of untracked node", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) return tracker - }(), - ip: newerIP, - expectedUpdated: true, - expectedState: func() *ipTracker { + }, + ip: newerIP, + expectedUpdatedAndDesired: false, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[newerIP.NodeID].ip = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + }, + }, + { + name: "connected newer IP of tracked node", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + return tracker + }, + ip: newerIP, + expectedUpdatedAndDesired: true, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - delete(tracker.gossipableIndices, newerIP.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] + tracker.subnet[constants.PrimaryNetworkID].gossipableIPs[0] = newerIP + }, + }, + { + name: "connected newer IP of untracked node", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + ip: newerIP, + expectedUpdatedAndDesired: false, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[newerIP.NodeID].ip = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - updated := test.initialState.AddIP(test.ip) - require.Equal(t, test.expectedUpdated, updated) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + updated := testState.AddIP(test.ip) + test.expectedChange(expectedState) + + require.Equal(t, test.expectedUpdatedAndDesired, updated) + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } func TestIPTracker_Connected(t *testing.T) { + subnetID := ids.GenerateTestID() newerIP := newerTestIP(ip) tests := []struct { - name string - initialState *ipTracker - ip *ips.ClaimedIPPort - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + ip *ips.ClaimedIPPort + expectedChange func(*ipTracker) }{ { name: "non-validator", - initialState: newTestIPTracker(t), + initialState: newTestIPTracker, ip: ip, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.connected[ip.NodeID] = ip - return tracker - }(), + expectedChange: func(tracker *ipTracker) { + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + }, }, { - name: "first known IP", - initialState: func() *ipTracker { + name: "first known IP of node tracking subnet", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, ip: ip, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() + tracker.tracked[ip.NodeID].ip = ip tracker.bloomAdditions[ip.NodeID] = 1 - tracker.connected[ip.NodeID] = ip - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[ip.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ ip, } + }, + }, + { + name: "first known IP of node not tracking subnet", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].ip = ip + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + }, }, { - name: "connected with older IP", - initialState: func() *ipTracker { + name: "connected with older IP of node tracking subnet", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker - }(), + }, ip: ip, - expectedState: func() *ipTracker { + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[newerIP.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ + newerIP, + } + }, + }, + { + name: "connected with older IP of node not tracking subnet", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(newerIP)) - tracker.connected[ip.NodeID] = ip + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(newerIP)) return tracker - }(), + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + }, }, { - name: "connected with newer IP", - initialState: func() *ipTracker { + name: "connected with newer IP of node tracking subnet", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: newerIP, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() + tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - tracker.connected[newerIP.NodeID] = newerIP - tracker.gossipableIndices[newerIP.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + tracker.connected[newerIP.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: newerIP, + } + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[newerIP.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ newerIP, } + }, + }, + { + name: "connected with newer IP of node not tracking subnet", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) return tracker - }(), + }, + ip: newerIP, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[newerIP.NodeID].ip = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + tracker.connected[newerIP.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: newerIP, + } + }, }, { - name: "connected with same IP", - initialState: func() *ipTracker { + name: "connected with same IP of node tracking subnet", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: ip, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - tracker.connected[ip.NodeID] = ip - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[ip.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ ip, } + }, + }, + { + name: "connected with same IP of node not tracking subnet", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) return tracker - }(), + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.Connected(test.ip) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.Connected(test.ip, set.Of(constants.PrimaryNetworkID)) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } func TestIPTracker_Disconnected(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + expectedChange func(*ipTracker) }{ - { - name: "not tracked", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip) - return tracker - }(), - nodeID: ip.NodeID, - expectedState: newTestIPTracker(t), - }, { name: "not gossipable", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.ManuallyTrack(ip.NodeID) - return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.ManuallyTrack(ip.NodeID) - delete(tracker.connected, ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + expectedChange: func(*ipTracker) {}, }, { name: "latest gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + }, + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Dec() delete(tracker.connected, ip.NodeID) - delete(tracker.gossipableIndices, ip.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] - return tracker - }(), + + subnet := tracker.subnet[constants.PrimaryNetworkID] + delete(subnet.gossipableIndices, ip.NodeID) + subnet.gossipableIPs = subnet.gossipableIPs[:0] + }, }, { name: "non-latest gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) + }, + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Dec() delete(tracker.connected, ip.NodeID) - tracker.gossipableIndices = map[ids.NodeID]int{ + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices = map[ids.NodeID]int{ otherIP.NodeID: 0, } - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + subnet.gossipableIPs = []*ips.ClaimedIPPort{ otherIP, } + }, + }, + { + name: "remove multiple gossipable IPs", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID, subnetID)) return tracker - }(), + }, + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Add(-2) + delete(tracker.connected, ip.NodeID) + + primarySubnet := tracker.subnet[constants.PrimaryNetworkID] + delete(primarySubnet.gossipableIndices, ip.NodeID) + primarySubnet.gossipableIPs = primarySubnet.gossipableIPs[:0] + + subnet := tracker.subnet[subnetID] + delete(subnet.gossipableIndices, ip.NodeID) + subnet.gossipableIPs = subnet.gossipableIPs[:0] + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.Disconnected(test.nodeID) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.Disconnected(ip.NodeID) + expectedState.Disconnected(ip.NodeID) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } func TestIPTracker_OnValidatorAdded(t *testing.T) { newerIP := newerTestIP(ip) - + subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + subnetID ids.ID + expectedChange func(*ipTracker) }{ { name: "manually tracked", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID].validatedSubnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } + }, + }, + { + name: "manually tracked and connected", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.gossipableIDs.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() + tracker.tracked[ip.NodeID].validatedSubnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + ip, + }, + } + }, }, { name: "manually tracked and connected with older IP", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) require.True(t, tracker.AddIP(newerIP)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip) - require.True(t, tracker.AddIP(newerIP)) - tracker.gossipableIDs.Add(ip.NodeID) - return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() + tracker.tracked[ip.NodeID].validatedSubnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + newerIP, + }, + } + }, }, { name: "manually gossiped", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) - return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(*ipTracker) {}, }, { name: "disconnected", - initialState: newTestIPTracker(t), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.trackedIDs.Add(ip.NodeID) - tracker.gossipableIDs.Add(ip.NodeID) - return tracker - }(), + initialState: newTestIPTracker, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.tracked[ip.NodeID] = &trackedNode{ + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + } + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } + }, }, { name: "connected", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip - tracker.trackedIDs.Add(ip.NodeID) + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } tracker.bloomAdditions[ip.NodeID] = 1 - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ - ip, + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + ip, + }, } - tracker.gossipableIDs.Add(ip.NodeID) + }, + }, + { + name: "connected to other subnet", + initialState: func(t *testing.T) *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + subnetID: subnetID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + validatedSubnets: set.Of(subnetID), + ip: ip, + } + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.subnet[subnetID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorAdded(test.nodeID, nil, ids.Empty, 0) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.OnValidatorAdded(test.subnetID, ip.NodeID, nil, ids.Empty, 0) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } func TestIPTracker_OnValidatorRemoved(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + subnetID ids.ID + expectedChange func(*ipTracker) }{ { - name: "manually tracked not gossipable", - initialState: func() *ipTracker { + name: "remove last validator of subnet", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Dec() + tracker.numTrackedSubnets.Dec() + delete(tracker.tracked, ip.NodeID) + delete(tracker.subnet, constants.PrimaryNetworkID) + }, + }, + { + name: "manually tracked not gossipable", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) - tracker.gossipableIDs.Remove(ip.NodeID) return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedSubnets.Dec() + + node := tracker.tracked[ip.NodeID] + node.validatedSubnets.Remove(constants.PrimaryNetworkID) + node.trackedSubnets.Remove(constants.PrimaryNetworkID) + + delete(tracker.subnet, constants.PrimaryNetworkID) + }, }, { name: "manually tracked latest gossipable", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - delete(tracker.gossipableIndices, ip.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] - tracker.gossipableIDs.Remove(ip.NodeID) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Dec() + tracker.numTrackedSubnets.Dec() + + node := tracker.tracked[ip.NodeID] + node.validatedSubnets.Remove(constants.PrimaryNetworkID) + node.trackedSubnets.Remove(constants.PrimaryNetworkID) + + delete(tracker.subnet, constants.PrimaryNetworkID) + }, }, { name: "manually gossiped", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(*ipTracker) {}, }, { - name: "not gossipable", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + name: "manually gossiped on other subnet", + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - delete(tracker.mostRecentTrackedIPs, ip.NodeID) - tracker.trackedIDs.Remove(ip.NodeID) - tracker.gossipableIDs.Remove(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - }, - { - name: "latest gossipable", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - delete(tracker.mostRecentTrackedIPs, ip.NodeID) - tracker.trackedIDs.Remove(ip.NodeID) - delete(tracker.gossipableIndices, ip.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] - tracker.gossipableIDs.Remove(ip.NodeID) - return tracker - }(), + }, + subnetID: subnetID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedSubnets.Dec() + tracker.tracked[ip.NodeID].validatedSubnets.Remove(subnetID) + delete(tracker.subnet, subnetID) + }, }, { name: "non-latest gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) - delete(tracker.mostRecentTrackedIPs, ip.NodeID) - tracker.trackedIDs.Remove(ip.NodeID) - tracker.gossipableIndices = map[ids.NodeID]int{ + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(tracker *ipTracker) { + tracker.numTrackedPeers.Dec() + tracker.numGossipableIPs.Dec() + delete(tracker.tracked, ip.NodeID) + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIDs.Remove(ip.NodeID) + subnet.gossipableIndices = map[ids.NodeID]int{ otherIP.NodeID: 0, } - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + subnet.gossipableIPs = []*ips.ClaimedIPPort{ otherIP, } - tracker.gossipableIDs.Remove(ip.NodeID) - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorRemoved(test.nodeID, 0) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) - }) - } -} - -func TestIPTracker_GetGossipableIPs(t *testing.T) { - require := require.New(t) - - tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.Connected(otherIP) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + testState := test.initialState(t) + expectedState := test.initialState(t) - gossipableIPs := tracker.GetGossipableIPs(ids.EmptyNodeID, bloom.EmptyFilter, nil, 2) - require.ElementsMatch([]*ips.ClaimedIPPort{ip, otherIP}, gossipableIPs) - - gossipableIPs = tracker.GetGossipableIPs(ip.NodeID, bloom.EmptyFilter, nil, 2) - require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) - - gossipableIPs = tracker.GetGossipableIPs(ids.EmptyNodeID, bloom.FullFilter, nil, 2) - require.Empty(gossipableIPs) - - filter, err := bloom.New(8, 1024) - require.NoError(err) - bloom.Add(filter, ip.GossipID[:], nil) - - readFilter, err := bloom.Parse(filter.Marshal()) - require.NoError(err) + testState.OnValidatorRemoved(test.subnetID, ip.NodeID, 0) + test.expectedChange(expectedState) - gossipableIPs = tracker.GetGossipableIPs(ip.NodeID, readFilter, nil, 2) - require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) -} - -func TestIPTracker_BloomFiltersEverything(t *testing.T) { - require := require.New(t) - - tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.Connected(otherIP) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) - - bloomBytes, salt := tracker.Bloom() - readFilter, err := bloom.Parse(bloomBytes) - require.NoError(err) - - gossipableIPs := tracker.GetGossipableIPs(ids.EmptyNodeID, readFilter, salt, 2) - require.Empty(gossipableIPs) - - require.NoError(tracker.ResetBloom()) + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) + }) + } } func TestIPTracker_BloomGrows(t *testing.T) { @@ -837,7 +1178,7 @@ func TestIPTracker_BloomGrows(t *testing.T) { { name: "Add Validator", add: func(tracker *ipTracker) { - tracker.OnValidatorAdded(ids.GenerateTestNodeID(), nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ids.GenerateTestNodeID(), nil, ids.Empty, 0) }, }, { @@ -846,6 +1187,12 @@ func TestIPTracker_BloomGrows(t *testing.T) { tracker.ManuallyTrack(ids.GenerateTestNodeID()) }, }, + { + name: "Manually Gossip", + add: func(tracker *ipTracker) { + tracker.ManuallyGossip(ids.GenerateTestID(), ids.GenerateTestNodeID()) + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -869,12 +1216,13 @@ func TestIPTracker_BloomResetsDynamically(t *testing.T) { require := require.New(t) tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorRemoved(ip.NodeID, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorRemoved(constants.PrimaryNetworkID, ip.NodeID, 0) + tracker.maxBloomCount = 1 - tracker.Connected(otherIP) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) requireMetricsConsistent(t, tracker) bloomBytes, salt := tracker.Bloom() @@ -892,7 +1240,7 @@ func TestIPTracker_PreventBloomFilterAddition(t *testing.T) { newestIP := newerTestIP(newerIP) tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) require.True(tracker.AddIP(ip)) require.True(tracker.AddIP(newerIP)) require.True(tracker.AddIP(newestIP)) @@ -900,18 +1248,133 @@ func TestIPTracker_PreventBloomFilterAddition(t *testing.T) { requireMetricsConsistent(t, tracker) } -func TestIPTracker_ShouldVerifyIP(t *testing.T) { - require := require.New(t) - - newerIP := newerTestIP(ip) +func TestIPTracker_GetGossipableIPs(t *testing.T) { + subnetIDA := ids.GenerateTestID() + subnetIDB := ids.GenerateTestID() + unknownSubnetID := ids.GenerateTestID() tracker := newTestIPTracker(t) - require.False(tracker.ShouldVerifyIP(ip)) - tracker.ManuallyTrack(ip.NodeID) - require.True(tracker.ShouldVerifyIP(ip)) - tracker.ManuallyGossip(ip.NodeID) - require.True(tracker.ShouldVerifyIP(ip)) - require.True(tracker.AddIP(ip)) - require.False(tracker.ShouldVerifyIP(ip)) - require.True(tracker.ShouldVerifyIP(newerIP)) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID, subnetIDA)) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID, subnetIDA, subnetIDB)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(subnetIDA, otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(subnetIDB, otherIP.NodeID, nil, ids.Empty, 0) + + myFilterBytes, mySalt := tracker.Bloom() + myFilter, err := bloom.Parse(myFilterBytes) + require.NoError(t, err) + + tests := []struct { + name string + toIterate set.Set[ids.ID] + allowed set.Set[ids.ID] + nodeID ids.NodeID + filter *bloom.ReadFilter + salt []byte + expected []*ips.ClaimedIPPort + }{ + { + name: "fetch both subnets IPs", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{ip, otherIP}, + }, + { + name: "filter nodeID", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA), + nodeID: ip.NodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter duplicate nodeIDs", + toIterate: set.Of(subnetIDA, subnetIDB), + allowed: set.Of(subnetIDA, subnetIDB), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter known IPs", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA), + nodeID: ids.EmptyNodeID, + filter: func() *bloom.ReadFilter { + filter, err := bloom.New(8, 1024) + require.NoError(t, err) + bloom.Add(filter, ip.GossipID[:], nil) + + readFilter, err := bloom.Parse(filter.Marshal()) + require.NoError(t, err) + return readFilter + }(), + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter everything", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA, subnetIDB), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA, subnetIDB), + nodeID: ids.EmptyNodeID, + filter: myFilter, + salt: mySalt, + expected: nil, + }, + { + name: "only fetch primary network IPs", + toIterate: set.Of(constants.PrimaryNetworkID), + allowed: set.Of(constants.PrimaryNetworkID), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{ip}, + }, + { + name: "only fetch subnet IPs", + toIterate: set.Of(subnetIDA), + allowed: set.Of(subnetIDA), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter subnet", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{ip}, + }, + { + name: "skip unknown subnet", + toIterate: set.Of(unknownSubnetID), + allowed: set.Of(unknownSubnetID), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: nil, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gossipableIPs := getGossipableIPs( + tracker, + test.toIterate, + test.allowed.Contains, + test.nodeID, + test.filter, + test.salt, + 2, + ) + require.ElementsMatch(t, test.expected, gossipableIPs) + }) + } } diff --git a/network/network.go b/network/network.go index 94993a129d81..9f785f34bd05 100644 --- a/network/network.go +++ b/network/network.go @@ -235,16 +235,16 @@ func NewNetwork( return nil, fmt.Errorf("initializing network metrics failed with: %w", err) } - ipTracker, err := newIPTracker(log, metricsRegisterer) + ipTracker, err := newIPTracker(config.TrackedSubnets, log, metricsRegisterer) if err != nil { return nil, fmt.Errorf("initializing ip tracker failed with: %w", err) } - config.Validators.RegisterSetCallbackListener(constants.PrimaryNetworkID, ipTracker) + config.Validators.RegisterCallbackListener(ipTracker) // Track all default bootstrappers to ensure their current IPs are gossiped // like validator IPs. for _, bootstrapper := range genesis.GetBootstrappers(config.NetworkID) { - ipTracker.ManuallyGossip(bootstrapper.ID) + ipTracker.ManuallyGossip(constants.PrimaryNetworkID, bootstrapper.ID) } // Track all recent validators to optimistically connect to them before the // P-chain has finished syncing. @@ -263,6 +263,7 @@ func NewNetwork( Network: nil, // This is set below. Router: router, VersionCompatibility: version.GetCompatibility(minCompatibleTime), + MyNodeID: config.MyNodeID, MySubnets: config.TrackedSubnets, Beacons: config.Beacons, Validators: config.Validators, @@ -454,14 +455,13 @@ func (n *network) Connected(nodeID ids.NodeID) { peerIP.Timestamp, peerIP.TLSSignature, ) - n.ipTracker.Connected(newIP) + trackedSubnets := peer.TrackedSubnets() + n.ipTracker.Connected(newIP, trackedSubnets) n.metrics.markConnected(peer) peerVersion := peer.Version() n.router.Connected(nodeID, peerVersion, constants.PrimaryNetworkID) - - trackedSubnets := peer.TrackedSubnets() for subnetID := range n.peerConfig.MySubnets { if trackedSubnets.Contains(subnetID) { n.router.Connected(nodeID, peerVersion, subnetID) @@ -477,13 +477,14 @@ func (n *network) AllowConnection(nodeID ids.NodeID) bool { if !n.config.RequireValidatorToConnect { return true } - _, iAmAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) - return iAmAValidator || n.ipTracker.WantsConnection(nodeID) + _, areWeAPrimaryNetworkAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) + return areWeAPrimaryNetworkAValidator || n.ipTracker.WantsConnection(nodeID) } func (n *network) Track(claimedIPPorts []*ips.ClaimedIPPort) error { + _, areWeAPrimaryNetworkAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) for _, ip := range claimedIPPorts { - if err := n.track(ip); err != nil { + if err := n.track(ip, areWeAPrimaryNetworkAValidator); err != nil { return err } } @@ -513,9 +514,61 @@ func (n *network) KnownPeers() ([]byte, []byte) { return n.ipTracker.Bloom() } -func (n *network) Peers(except ids.NodeID, knownPeers *bloom.ReadFilter, salt []byte) []*ips.ClaimedIPPort { - return n.ipTracker.GetGossipableIPs( - except, +// There are 3 types of responses: +// +// - Respond with subnet IPs tracked by both ourselves and the peer +// - We do not consider ourself to be a primary network validator +// +// - Respond with all subnet IPs +// - The peer requests all peers +// - We believe the peer to be a primary network validator +// - We believe ourself to be a primary network validator +// +// - Respond with subnet IPs tracked by the peer +// - Either the peer does not request all peers or we don't consider them to +// be a primary network validator +// - We believe ourself to be a primary network validator +// +// The reason we allow the peer to request all peers is so that we can avoid +// sending unnecessary data in the case that we consider them a primary network +// validator but they do not consider themselves one. +func (n *network) Peers( + peerID ids.NodeID, + trackedSubnets set.Set[ids.ID], + requestAllPeers bool, + knownPeers *bloom.ReadFilter, + salt []byte, +) []*ips.ClaimedIPPort { + _, areWeAPrimaryNetworkValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) + + // Only return IPs for subnets that we are tracking. + var allowedSubnets func(ids.ID) bool + if areWeAPrimaryNetworkValidator { + allowedSubnets = func(ids.ID) bool { return true } + } else { + allowedSubnets = func(subnetID ids.ID) bool { + return subnetID == constants.PrimaryNetworkID || n.ipTracker.trackedSubnets.Contains(subnetID) + } + } + + _, areTheyAPrimaryNetworkValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, peerID) + if areWeAPrimaryNetworkValidator && requestAllPeers && areTheyAPrimaryNetworkValidator { + // Return IPs for all subnets. + return getGossipableIPs( + n.ipTracker, + n.ipTracker.subnet, + allowedSubnets, + peerID, + knownPeers, + salt, + int(n.config.PeerListNumValidatorIPs), + ) + } + return getGossipableIPs( + n.ipTracker, + trackedSubnets, + allowedSubnets, + peerID, knownPeers, salt, int(n.config.PeerListNumValidatorIPs), @@ -621,7 +674,7 @@ func (n *network) ManuallyTrack(nodeID ids.NodeID, ip netip.AddrPort) { } } -func (n *network) track(ip *ips.ClaimedIPPort) error { +func (n *network) track(ip *ips.ClaimedIPPort, trackAllSubnets bool) error { // To avoid signature verification when the IP isn't needed, we // optimistically filter out IPs. This can result in us not tracking an IP // that we otherwise would have. This case can only happen if the node @@ -630,7 +683,7 @@ func (n *network) track(ip *ips.ClaimedIPPort) error { // // Note: Avoiding signature verification when the IP isn't needed is a // **significant** performance optimization. - if !n.ipTracker.ShouldVerifyIP(ip) { + if !n.ipTracker.ShouldVerifyIP(ip, trackAllSubnets) { n.metrics.numUselessPeerListBytes.Add(float64(ip.Size())) return nil } @@ -698,9 +751,9 @@ func (n *network) getPeers( continue } - _, isValidator := n.config.Validators.GetValidator(subnetID, nodeID) + _, areTheyAValidator := n.config.Validators.GetValidator(subnetID, nodeID) // check if the peer is allowed to connect to the subnet - if !allower.IsAllowed(nodeID, isValidator) { + if !allower.IsAllowed(nodeID, areTheyAValidator) { continue } @@ -741,9 +794,9 @@ func (n *network) samplePeers( return false } - _, isValidator := n.config.Validators.GetValidator(subnetID, peerID) + _, areTheyAValidator := n.config.Validators.GetValidator(subnetID, peerID) // check if the peer is allowed to connect to the subnet - if !allower.IsAllowed(peerID, isValidator) { + if !allower.IsAllowed(peerID, areTheyAValidator) { return false } @@ -752,7 +805,7 @@ func (n *network) samplePeers( return true } - if isValidator { + if areTheyAValidator { numValidatorsToSample-- return numValidatorsToSample >= 0 } diff --git a/network/peer/config.go b/network/peer/config.go index 8aa12820cc41..4ee52ecf48e0 100644 --- a/network/peer/config.go +++ b/network/peer/config.go @@ -33,6 +33,7 @@ type Config struct { Network Network Router router.InboundHandler VersionCompatibility version.Compatibility + MyNodeID ids.NodeID // MySubnets does not include the primary network ID MySubnets set.Set[ids.ID] Beacons validators.Manager diff --git a/network/peer/network.go b/network/peer/network.go index b8fb01814546..19161321f1c3 100644 --- a/network/peer/network.go +++ b/network/peer/network.go @@ -7,6 +7,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/ips" + "github.com/ava-labs/avalanchego/utils/set" ) // Network defines the interface that is used by a peer to help establish a well @@ -35,6 +36,8 @@ type Network interface { // Peers returns peers that are not known. Peers( peerID ids.NodeID, + trackedSubnets set.Set[ids.ID], + requestAllPeers bool, knownPeers *bloom.ReadFilter, peerSalt []byte, ) []*ips.ClaimedIPPort diff --git a/network/peer/peer.go b/network/peer/peer.go index a92791ff72ee..c424c1fbdccd 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -534,6 +534,7 @@ func (p *peer) writeMessages() { myVersion := p.VersionCompatibility.Version() knownPeersFilter, knownPeersSalt := p.Network.KnownPeers() + _, areWeAPrimaryNetworkValidator := p.Validators.GetValidator(constants.PrimaryNetworkID, p.MyNodeID) msg, err := p.MessageCreator.Handshake( p.NetworkID, p.Clock.Unix(), @@ -550,6 +551,7 @@ func (p *peer) writeMessages() { p.ObjectedACPs, knownPeersFilter, knownPeersSalt, + areWeAPrimaryNetworkValidator, ) if err != nil { p.Log.Error(failedToCreateMessageLog, @@ -643,7 +645,12 @@ func (p *peer) sendNetworkMessages() { select { case <-p.getPeerListChan: knownPeersFilter, knownPeersSalt := p.Config.Network.KnownPeers() - msg, err := p.Config.MessageCreator.GetPeerList(knownPeersFilter, knownPeersSalt) + _, areWeAPrimaryNetworkValidator := p.Validators.GetValidator(constants.PrimaryNetworkID, p.MyNodeID) + msg, err := p.Config.MessageCreator.GetPeerList( + knownPeersFilter, + knownPeersSalt, + areWeAPrimaryNetworkValidator, + ) if err != nil { p.Log.Error(failedToCreateMessageLog, zap.Stringer("nodeID", p.id), @@ -1113,7 +1120,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.gotHandshake.Set(true) - peerIPs := p.Network.Peers(p.id, knownPeers, salt) + peerIPs := p.Network.Peers(p.id, p.trackedSubnets, msg.AllSubnets, knownPeers, salt) // We bypass throttling here to ensure that the handshake message is // acknowledged correctly. @@ -1175,7 +1182,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { return } - peerIPs := p.Network.Peers(p.id, filter, salt) + peerIPs := p.Network.Peers(p.id, p.trackedSubnets, msg.AllSubnets, filter, salt) if len(peerIPs) == 0 { p.Log.Debug("skipping sending of empty peer list", zap.Stringer("nodeID", p.id), diff --git a/network/peer/peer_test.go b/network/peer/peer_test.go index 316b491180af..0ae75b571f27 100644 --- a/network/peer/peer_test.go +++ b/network/peer/peer_test.go @@ -42,7 +42,6 @@ type testPeer struct { type rawTestPeer struct { config *Config cert *staking.Certificate - nodeID ids.NodeID inboundMsgChan <-chan message.InboundMessage } @@ -106,7 +105,7 @@ func newRawTestPeer(t *testing.T, config Config) *rawTestPeer { require.NoError(err) cert, err := staking.ParseCertificate(tlsCert.Leaf.Raw) require.NoError(err) - nodeID := ids.NodeIDFromCert(cert) + config.MyNodeID = ids.NodeIDFromCert(cert) ip := utils.NewAtomic(netip.AddrPortFrom( netip.IPv6Loopback(), @@ -126,7 +125,6 @@ func newRawTestPeer(t *testing.T, config Config) *rawTestPeer { return &rawTestPeer{ config: &config, cert: cert, - nodeID: nodeID, inboundMsgChan: inboundMsgChan, } } @@ -137,10 +135,10 @@ func startTestPeer(self *rawTestPeer, peer *rawTestPeer, conn net.Conn) *testPee self.config, conn, peer.cert, - peer.nodeID, + peer.config.MyNodeID, NewThrottledMessageQueue( self.config.Metrics, - peer.nodeID, + peer.config.MyNodeID, logging.NoLog{}, throttling.NewNoOutboundThrottler(), ), @@ -411,7 +409,7 @@ func TestInvalidBLSKeyDisconnects(t *testing.T) { require.NoError(rawPeer0.config.Validators.AddStaker( constants.PrimaryNetworkID, - rawPeer1.nodeID, + rawPeer1.config.MyNodeID, bls.PublicFromSecretKey(rawPeer1.config.IPSigner.blsSigner), ids.GenerateTestID(), 1, @@ -421,7 +419,7 @@ func TestInvalidBLSKeyDisconnects(t *testing.T) { require.NoError(err) require.NoError(rawPeer1.config.Validators.AddStaker( constants.PrimaryNetworkID, - rawPeer0.nodeID, + rawPeer0.config.MyNodeID, bls.PublicFromSecretKey(bogusBLSKey), // This is the wrong BLS key for this peer ids.GenerateTestID(), 1, diff --git a/network/peer/test_network.go b/network/peer/test_network.go index 01a341ae9abc..b38c7788081e 100644 --- a/network/peer/test_network.go +++ b/network/peer/test_network.go @@ -7,6 +7,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/ips" + "github.com/ava-labs/avalanchego/utils/set" ) var TestNetwork Network = testNetwork{} @@ -29,6 +30,12 @@ func (testNetwork) KnownPeers() ([]byte, []byte) { return bloom.EmptyFilter.Marshal(), nil } -func (testNetwork) Peers(ids.NodeID, *bloom.ReadFilter, []byte) []*ips.ClaimedIPPort { +func (testNetwork) Peers( + ids.NodeID, + set.Set[ids.ID], + bool, + *bloom.ReadFilter, + []byte, +) []*ips.ClaimedIPPort { return nil } diff --git a/proto/p2p/p2p.proto b/proto/p2p/p2p.proto index 7935ef6e44a5..7df36717c56e 100644 --- a/proto/p2p/p2p.proto +++ b/proto/p2p/p2p.proto @@ -112,6 +112,9 @@ message Handshake { // Signature of the peer IP port pair at a provided timestamp with the BLS // key. bytes ip_bls_sig = 13; + // To avoid sending IPs that the client isn't interested in tracking, the + // server expects the client to confirm that it is tracking all subnets. + bool all_subnets = 14; } // Metadata about a peer's P2P client used to determine compatibility @@ -154,6 +157,7 @@ message ClaimedIpPort { // filter. message GetPeerList { BloomFilter known_peers = 1; + bool all_subnets = 2; } // PeerList contains network-level metadata for a set of validators. diff --git a/proto/pb/p2p/p2p.pb.go b/proto/pb/p2p/p2p.pb.go index df754b0e2d74..9b10923bbf20 100644 --- a/proto/pb/p2p/p2p.pb.go +++ b/proto/pb/p2p/p2p.pb.go @@ -679,6 +679,9 @@ type Handshake struct { // Signature of the peer IP port pair at a provided timestamp with the BLS // key. IpBlsSig []byte `protobuf:"bytes,13,opt,name=ip_bls_sig,json=ipBlsSig,proto3" json:"ip_bls_sig,omitempty"` + // To avoid sending IPs that the client isn't interested in tracking, the + // server expects the client to confirm that it is tracking all subnets. + AllSubnets bool `protobuf:"varint,14,opt,name=all_subnets,json=allSubnets,proto3" json:"all_subnets,omitempty"` } func (x *Handshake) Reset() { @@ -797,6 +800,13 @@ func (x *Handshake) GetIpBlsSig() []byte { return nil } +func (x *Handshake) GetAllSubnets() bool { + if x != nil { + return x.AllSubnets + } + return false +} + // Metadata about a peer's P2P client used to determine compatibility type Client struct { state protoimpl.MessageState @@ -1033,6 +1043,7 @@ type GetPeerList struct { unknownFields protoimpl.UnknownFields KnownPeers *BloomFilter `protobuf:"bytes,1,opt,name=known_peers,json=knownPeers,proto3" json:"known_peers,omitempty"` + AllSubnets bool `protobuf:"varint,2,opt,name=all_subnets,json=allSubnets,proto3" json:"all_subnets,omitempty"` } func (x *GetPeerList) Reset() { @@ -1074,6 +1085,13 @@ func (x *GetPeerList) GetKnownPeers() *BloomFilter { return nil } +func (x *GetPeerList) GetAllSubnets() bool { + if x != nil { + return x.AllSubnets + } + return false +} + // PeerList contains network-level metadata for a set of validators. // // PeerList must be sent in response to an inbound Handshake message from a @@ -2640,7 +2658,7 @@ var file_p2p_p2p_proto_rawDesc = []byte{ 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x6e, - 0x67, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0xb3, 0x03, + 0x67, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0xd4, 0x03, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6d, 0x79, @@ -2667,202 +2685,206 @@ var file_p2p_p2p_proto_rawDesc = []byte{ 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x0a, 0x69, 0x70, 0x5f, 0x62, 0x6c, 0x73, 0x5f, 0x73, 0x69, 0x67, 0x18, 0x0d, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x08, 0x69, 0x70, 0x42, 0x6c, 0x73, 0x53, 0x69, 0x67, 0x4a, 0x04, 0x08, - 0x05, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x14, 0x0a, - 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x70, 0x61, - 0x74, 0x63, 0x68, 0x22, 0x39, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x61, - 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x22, 0xbd, - 0x01, 0x0a, 0x0d, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x29, 0x0a, 0x10, 0x78, 0x35, 0x30, 0x39, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x78, 0x35, 0x30, 0x39, - 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, - 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, 0x70, - 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x78, 0x5f, - 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x78, 0x49, 0x64, 0x22, 0x40, - 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x31, 0x0a, - 0x0b, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x73, - 0x22, 0x48, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x10, - 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x5f, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, 0x6c, 0x61, - 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x52, 0x0e, 0x63, 0x6c, 0x61, 0x69, - 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x6f, 0x0a, 0x17, 0x47, 0x65, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, + 0x01, 0x28, 0x0c, 0x52, 0x08, 0x69, 0x70, 0x42, 0x6c, 0x73, 0x53, 0x69, 0x67, 0x12, 0x1f, 0x0a, + 0x0b, 0x61, 0x6c, 0x6c, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x4a, 0x04, + 0x08, 0x05, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x6f, + 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x14, + 0x0a, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x70, + 0x61, 0x74, 0x63, 0x68, 0x22, 0x39, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x73, + 0x61, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x22, + 0xbd, 0x01, 0x0a, 0x0d, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, + 0x74, 0x12, 0x29, 0x0a, 0x10, 0x78, 0x35, 0x30, 0x39, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x78, 0x35, 0x30, + 0x39, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x07, + 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, + 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x78, + 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x78, 0x49, 0x64, 0x22, + 0x61, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x31, + 0x0a, 0x0b, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, + 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x6c, 0x6c, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x53, 0x75, 0x62, 0x6e, 0x65, + 0x74, 0x73, 0x22, 0x48, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3c, + 0x0a, 0x10, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x5f, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, + 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x52, 0x0e, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x6f, 0x0a, 0x17, + 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, + 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x6a, 0x0a, + 0x14, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x89, 0x01, 0x0a, 0x17, 0x47, 0x65, + 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, + 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, + 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x6a, 0x0a, 0x14, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, - 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, - 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, - 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x89, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x41, - 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, - 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, - 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, - 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x69, - 0x67, 0x68, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x68, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x73, 0x22, 0x71, 0x0a, 0x14, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, - 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, 0x6d, 0x6d, - 0x61, 0x72, 0x79, 0x49, 0x64, 0x73, 0x22, 0x71, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, - 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, - 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x6f, 0x0a, 0x10, 0x41, 0x63, 0x63, - 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, + 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x68, 0x65, + 0x69, 0x67, 0x68, 0x74, 0x73, 0x22, 0x71, 0x0a, 0x14, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8e, 0x01, 0x0a, 0x0b, 0x47, - 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, - 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x49, 0x64, 0x73, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x69, 0x0a, 0x08, 0x41, - 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, 0x22, 0xb9, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x6e, - 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, - 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x22, 0x65, 0x0a, 0x09, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x6d, 0x6d, 0x61, + 0x72, 0x79, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, + 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x49, 0x64, 0x73, 0x22, 0x71, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, + 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x03, 0x47, 0x65, - 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x6f, 0x0a, 0x10, 0x41, + 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8e, 0x01, 0x0a, + 0x0b, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, + 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, + 0x6e, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x69, 0x0a, + 0x08, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, 0x22, 0xb9, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, + 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, + 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, + 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x22, 0x65, 0x0a, 0x09, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, + 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, - 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, - 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, - 0x22, 0x5d, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, + 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x03, + 0x47, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, + 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x05, + 0x10, 0x06, 0x22, 0x5d, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x22, 0xb0, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, + 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, + 0x08, 0x05, 0x10, 0x06, 0x22, 0xb5, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, + 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, + 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, + 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, + 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0xba, 0x01, 0x0a, + 0x05, 0x43, 0x68, 0x69, 0x74, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, + 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, + 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, + 0x64, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, + 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, + 0x65, 0x64, 0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, + 0x64, 0x5f, 0x69, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, + 0x64, 0x41, 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x7f, 0x0a, 0x0a, 0x41, 0x70, 0x70, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x22, - 0xb0, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x19, 0x0a, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1b, 0x0a, + 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0b, 0x41, 0x70, + 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x22, 0x88, 0x01, 0x0a, 0x08, 0x41, 0x70, 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, - 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, 0x08, 0x05, - 0x10, 0x06, 0x22, 0xb5, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, - 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, - 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, - 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, - 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0xba, 0x01, 0x0a, 0x05, 0x43, - 0x68, 0x69, 0x74, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, - 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, - 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, - 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x64, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, - 0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, - 0x69, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x13, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, 0x64, 0x41, - 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x7f, 0x0a, 0x0a, 0x41, 0x70, 0x70, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, - 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, - 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x61, - 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, - 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0b, 0x41, 0x70, 0x70, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x88, - 0x01, 0x0a, 0x08, 0x41, 0x70, 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, - 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x11, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x43, 0x6f, 0x64, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x43, 0x0a, 0x09, 0x41, 0x70, 0x70, - 0x47, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0x5d, - 0x0a, 0x0a, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, - 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x4e, 0x47, - 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x56, 0x41, 0x4c, 0x41, 0x4e, 0x43, - 0x48, 0x45, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4e, 0x4f, 0x57, 0x4d, 0x41, 0x4e, 0x10, 0x02, 0x42, 0x2e, 0x5a, - 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x76, 0x61, 0x2d, - 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x61, 0x76, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x67, 0x6f, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x2f, 0x70, 0x32, 0x70, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x11, 0x52, 0x09, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x43, 0x0a, 0x09, 0x41, + 0x70, 0x70, 0x47, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x2a, 0x5d, 0x0a, 0x0a, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, + 0x0a, 0x17, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x45, + 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x56, 0x41, 0x4c, 0x41, + 0x4e, 0x43, 0x48, 0x45, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4e, 0x4f, 0x57, 0x4d, 0x41, 0x4e, 0x10, 0x02, 0x42, + 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x76, + 0x61, 0x2d, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x61, 0x76, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x68, 0x65, + 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x2f, 0x70, 0x32, 0x70, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var (