diff --git a/pkg/gossip/gossip.go b/pkg/gossip/gossip.go index 290f529497dd..434aad1d993f 100644 --- a/pkg/gossip/gossip.go +++ b/pkg/gossip/gossip.go @@ -151,6 +151,22 @@ var ( defaultGossipStoresInterval) ) +// KeyNotPresentError is returned by gossip when queried for a key that doesn't +// exist of has expired. +type KeyNotPresentError struct { + key string +} + +// Error implements the error interface. +func (err KeyNotPresentError) Error() string { + return fmt.Sprintf("KeyNotPresentError: gossip key %q does not exist or has expired", err.key) +} + +// NewKeyNotPresentError creates a new KeyNotPresentError. +func NewKeyNotPresentError(key string) error { + return KeyNotPresentError{key: key} +} + // Storage is an interface which allows the gossip instance // to read and write bootstrapping data to persistent storage // between instantiations. @@ -788,7 +804,7 @@ func (g *Gossip) AddInfoProto(key string, msg proto.Message, ttl time.Duration) return g.AddInfo(key, bytes, ttl) } -// GetInfo returns an info value by key or an error if specified +// GetInfo returns an info value by key or an KeyNotPresentError if specified // key does not exist or has expired. func (g *Gossip) GetInfo(key string) ([]byte, error) { g.mu.Lock() @@ -801,10 +817,10 @@ func (g *Gossip) GetInfo(key string) ([]byte, error) { } return i.Value.GetBytes() } - return nil, errors.Errorf("key %q does not exist or has expired", key) + return nil, NewKeyNotPresentError(key) } -// GetInfoProto returns an info value by key or an error if specified +// GetInfoProto returns an info value by key or KeyNotPresentError if specified // key does not exist or has expired. func (g *Gossip) GetInfoProto(key string, msg proto.Message) error { bytes, err := g.GetInfo(key) diff --git a/pkg/gossip/keys.go b/pkg/gossip/keys.go index 6a25561f5407..ef89d043b98d 100644 --- a/pkg/gossip/keys.go +++ b/pkg/gossip/keys.go @@ -72,6 +72,10 @@ const ( // The value if a config.SystemConfig which holds all key/value // pairs in the system DB span. KeySystemConfig = "system-db" + + // KeyDistSQLNodeVersionKeyPrefix is key prefix for each node's DistSQL + // version. + KeyDistSQLNodeVersionKeyPrefix = "distsql-version" ) // MakeKey creates a canonical key under which to gossip a piece of @@ -128,3 +132,8 @@ func MakeStoreKey(storeID roachpb.StoreID) string { func MakeDeadReplicasKey(storeID roachpb.StoreID) string { return MakeKey(KeyDeadReplicasPrefix, storeID.String()) } + +// MakeDistSQLNodeVersionKey returns the gossip key for the given store. +func MakeDistSQLNodeVersionKey(nodeID roachpb.NodeID) string { + return MakeKey(KeyDistSQLNodeVersionKeyPrefix, nodeID.String()) +} diff --git a/pkg/server/server.go b/pkg/server/server.go index d9ad4133995e..68b5e34289fa 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -359,6 +359,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { Metrics: &distSQLMetrics, JobRegistry: s.jobRegistry, + Gossip: s.gossip, } if distSQLTestingKnobs := s.cfg.TestingKnobs.DistSQL; distSQLTestingKnobs != nil { distSQLCfg.TestingKnobs = *distSQLTestingKnobs.(*distsqlrun.TestingKnobs) diff --git a/pkg/sql/distsql_physical_planner.go b/pkg/sql/distsql_physical_planner.go index 1668cc2fb6cd..4a2efa21c986 100644 --- a/pkg/sql/distsql_physical_planner.go +++ b/pkg/sql/distsql_physical_planner.go @@ -60,6 +60,11 @@ import ( // and add processing stages (connected to the result routers of the children // node). type distSQLPlanner struct { + // planVersion is the version of DistSQL targeted by the plan we're building. + // This is currently only assigned to the node's current DistSQL version and + // is used to skip incompatible nodes when mapping spans. + planVersion distsqlrun.DistSQLVersion + st *cluster.Settings // The node descriptor for the gateway node that initiated this query. nodeDesc roachpb.NodeDescriptor @@ -72,6 +77,9 @@ type distSQLPlanner struct { // runnerChan is used to send out requests (for running SetupFlow RPCs) to a // pool of workers. runnerChan chan runnerRequest + + // gossip handle use to check node version compatibility + gossip *gossip.Gossip } const resolverPolicy = distsqlplan.BinPackingLeaseHolderChoice @@ -81,6 +89,7 @@ const resolverPolicy = distsqlplan.BinPackingLeaseHolderChoice var logPlanDiagram = envutil.EnvOrDefaultBool("COCKROACH_DISTSQL_LOG_PLAN", false) func newDistSQLPlanner( + planVersion distsqlrun.DistSQLVersion, st *cluster.Settings, nodeDesc roachpb.NodeDescriptor, rpcCtx *rpc.Context, @@ -91,11 +100,13 @@ func newDistSQLPlanner( testingKnobs DistSQLPlannerTestingKnobs, ) *distSQLPlanner { dsp := &distSQLPlanner{ + planVersion: planVersion, st: st, nodeDesc: nodeDesc, rpcContext: rpcCtx, stopper: stopper, distSQLSrv: distSQLSrv, + gossip: gossip, spanResolver: distsqlplan.NewSpanResolver(distSender, gossip, nodeDesc, resolverPolicy), testingKnobs: testingKnobs, } @@ -412,6 +423,10 @@ type spanPartition struct { // spanPartitions (one for each relevant node), which form a partitioning of the // spans (i.e. they are non-overlapping and their union is exactly the original // set of spans). +// +// partitionSpans does its best to not assign ranges on nodes that are known to +// either be unhealthy or running an incompatible version. The ranges owned by +// such nodes are assigned to the gateway. func (dsp *distSQLPlanner) partitionSpans( planCtx *planningCtx, spans roachpb.Spans, ) ([]spanPartition, error) { @@ -422,6 +437,9 @@ func (dsp *distSQLPlanner) partitionSpans( partitions := make([]spanPartition, 0, 1) // nodeMap maps a nodeID to an index inside the partitions array. nodeMap := make(map[roachpb.NodeID]int) + // nodeVerCompatMap maintains info about which nodes advertise DistSQL + // versions compatible with this plan and which ones don't. + nodeVerCompatMap := make(map[roachpb.NodeID]bool) it := planCtx.spanIter for _, span := range spans { var rspan roachpb.RSpan @@ -489,12 +507,26 @@ func (dsp *distSQLPlanner) partitionSpans( } planCtx.nodeAddresses[nodeID] = addr } - if addr == "" { - // An empty address indicates an unhealthy host. Use the gateway to - // process this span instead of the unhealthy host. + var compat bool + if addr != "" { + // Check if the node's DistSQL version is compatible with this plan. + // If it isn't, we'll use the gateway. + var ok bool + if compat, ok = nodeVerCompatMap[nodeID]; !ok { + compat = dsp.nodeVersionIsCompatible(nodeID, dsp.planVersion) + nodeVerCompatMap[nodeID] = compat + } + } + // If the node is unhealthy or its DistSQL version is incompatible, use + // the gateway to process this span instead of the unhealthy host. + // An empty address indicates an unhealthy host. + if addr == "" || !compat { + log.Eventf(ctx, "not planning on node %d. unhealthy: %t, incompatible version: %t", + nodeID, addr == "", !compat) nodeID = dsp.nodeDesc.NodeID partitionIdx, inNodeMap = nodeMap[nodeID] } + if !inNodeMap { partitionIdx = len(partitions) partitions = append(partitions, spanPartition{node: nodeID}) @@ -525,6 +557,25 @@ func (dsp *distSQLPlanner) partitionSpans( return partitions, nil } +// nodeVersionIsCompatible decides whether a particular node's DistSQL version +// is compatible with planVer. It uses gossip to find out the node's version +// range. +func (dsp *distSQLPlanner) nodeVersionIsCompatible( + nodeID roachpb.NodeID, planVer distsqlrun.DistSQLVersion, +) bool { + var v distsqlrun.DistSQLVersionGossipInfo + if hook := dsp.testingKnobs.OverrideDistSQLVersionCheck; hook != nil { + if err := hook(nodeID, &v); err != nil { + return false + } + } else { + if err := dsp.gossip.GetInfoProto(gossip.MakeDistSQLNodeVersionKey(nodeID), &v); err != nil { + return false + } + } + return distsqlrun.FlowVerIsCompatible(dsp.planVersion, v.MinAcceptedVersion, v.Version) +} + // initTableReaderSpec initializes a TableReaderSpec/PostProcessSpec that // corresponds to a scanNode, except for the Spans and OutputColumns. func initTableReaderSpec( diff --git a/pkg/sql/distsql_physical_planner_test.go b/pkg/sql/distsql_physical_planner_test.go index c1be8d6f0f6d..4cfedcffecc2 100644 --- a/pkg/sql/distsql_physical_planner_test.go +++ b/pkg/sql/distsql_physical_planner_test.go @@ -29,6 +29,7 @@ import ( "golang.org/x/net/context" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -636,7 +637,7 @@ func TestPartitionSpans(t *testing.T) { // spans to be passed to partitionSpans spans [][2]string - // expected result: a list of spans, one for each node. + // expected result: a map of node to list of spans. partitions map[int][][2]string }{ { @@ -736,6 +737,7 @@ func TestPartitionSpans(t *testing.T) { tsp.ranges = tc.ranges dsp := distSQLPlanner{ + planVersion: distsqlrun.Version, st: cluster.MakeTestingClusterSettings(), nodeDesc: *tsp.nodes[tc.gatewayNode-1], stopper: stopper, @@ -749,6 +751,14 @@ func TestPartitionSpans(t *testing.T) { } return nil }, + OverrideDistSQLVersionCheck: func( + node roachpb.NodeID, res *distsqlrun.DistSQLVersionGossipInfo, + ) error { + // Accept any version. + res.MinAcceptedVersion = 0 + res.Version = 1000000 + return nil + }, }, } @@ -781,3 +791,158 @@ func TestPartitionSpans(t *testing.T) { }) } } + +// Test that span partitioning takes into account the advertised acceptable +// versions of each node. Spans for which the owner node doesn't support our +// plan's version will be planned on the gateway. +func TestPartitionSpansSkipsIncompatibleNodes(t *testing.T) { + defer leaktest.AfterTest(t)() + + // The spans that we're going to plan for. + span := roachpb.Span{Key: roachpb.Key("A"), EndKey: roachpb.Key("Z")} + gatewayNode := roachpb.NodeID(2) + ranges := []testSpanResolverRange{{"A", 1}, {"B", 2}, {"C", 1}} + + testCases := []struct { + // the test's name + name string + + // planVersion is the DistSQL version that this plan is targeting. + // We'll play with this version and expect nodes to be skipped because of + // this. + planVersion distsqlrun.DistSQLVersion + + // The versions accepted by each node. + nodeVersions map[roachpb.NodeID]distsqlrun.DistSQLVersionGossipInfo + + // nodesNotInGossip is the set of nodes for which we're going to pretend + // that gossip returns an error when queried about the DistSQL version. + nodesNotInGossip map[roachpb.NodeID]struct{} + + // expected result: a map of node to list of spans. + partitions map[roachpb.NodeID][][2]string + }{ + { + // In the first test, all nodes are compatible. + name: "current_version", + planVersion: 2, + nodeVersions: map[roachpb.NodeID]distsqlrun.DistSQLVersionGossipInfo{ + 1: { + MinAcceptedVersion: 1, + Version: 2, + }, + 2: { + MinAcceptedVersion: 1, + Version: 2, + }, + }, + partitions: map[roachpb.NodeID][][2]string{ + 1: {{"A", "B"}, {"C", "Z"}}, + 2: {{"B", "C"}}, + }, + }, + { + // Plan version is incompatible with node 1. We expect everything to be + // assigned to the gateway. + // Remember that the gateway is node 2. + name: "next_version", + planVersion: 3, + nodeVersions: map[roachpb.NodeID]distsqlrun.DistSQLVersionGossipInfo{ + 1: { + MinAcceptedVersion: 1, + Version: 2, + }, + 2: { + MinAcceptedVersion: 3, + Version: 3, + }, + }, + partitions: map[roachpb.NodeID][][2]string{ + 2: {{"A", "Z"}}, + }, + }, + { + // Like the above, except node 1 is not gossiping its version (simulating + // a crdb 1.0 node). + name: "crdb_1.0", + planVersion: 3, + nodeVersions: map[roachpb.NodeID]distsqlrun.DistSQLVersionGossipInfo{ + 2: { + MinAcceptedVersion: 3, + Version: 3, + }, + }, + nodesNotInGossip: map[roachpb.NodeID]struct{}{ + 1: {}, + }, + partitions: map[roachpb.NodeID][][2]string{ + 2: {{"A", "Z"}}, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + stopper := stop.NewStopper() + defer stopper.Stop(context.TODO()) + + tsp := &testSpanResolver{} + for i := 1; i <= 2; i++ { + tsp.nodes = append(tsp.nodes, &roachpb.NodeDescriptor{ + NodeID: roachpb.NodeID(i), + Address: util.UnresolvedAddr{ + AddressField: fmt.Sprintf("addr%d", i), + }, + }) + } + tsp.ranges = ranges + + dsp := distSQLPlanner{ + planVersion: tc.planVersion, + st: cluster.MakeTestingClusterSettings(), + nodeDesc: *tsp.nodes[gatewayNode-1], + stopper: stopper, + spanResolver: tsp, + testingKnobs: DistSQLPlannerTestingKnobs{ + OverrideHealthCheck: func(node roachpb.NodeID, addr string) error { + // All the nodes are healthy. + return nil + }, + OverrideDistSQLVersionCheck: func( + nodeID roachpb.NodeID, res *distsqlrun.DistSQLVersionGossipInfo, + ) error { + if _, ok := tc.nodesNotInGossip[nodeID]; ok { + return gossip.NewKeyNotPresentError( + gossip.MakeDistSQLNodeVersionKey(nodeID), + ) + } + *res = tc.nodeVersions[nodeID] + return nil + }, + }, + } + + planCtx := dsp.NewPlanningCtx(context.Background(), nil /* txn */) + partitions, err := dsp.partitionSpans(&planCtx, roachpb.Spans{span}) + if err != nil { + t.Fatal(err) + } + + resMap := make(map[roachpb.NodeID][][2]string) + for _, p := range partitions { + if _, ok := resMap[p.node]; ok { + t.Fatalf("node %d shows up in multiple partitions", p) + } + var spans [][2]string + for _, s := range p.spans { + spans = append(spans, [2]string{string(s.Key), string(s.EndKey)}) + } + resMap[p.node] = spans + } + + if !reflect.DeepEqual(resMap, tc.partitions) { + t.Errorf("expected partitions:\n %v\ngot:\n %v", tc.partitions, resMap) + } + }) + } +} diff --git a/pkg/sql/distsqlrun/api.pb.go b/pkg/sql/distsqlrun/api.pb.go index 10bbbbb36f3a..fa5e9a949ead 100644 --- a/pkg/sql/distsqlrun/api.pb.go +++ b/pkg/sql/distsqlrun/api.pb.go @@ -27,6 +27,7 @@ ProducerData ProducerMessage RemoteProducerMetadata + DistSQLVersionGossipInfo ProcessorSpec PostProcessSpec ProcessorCoreUnion @@ -76,9 +77,9 @@ type SetupFlowRequest struct { Txn cockroach_roachpb1.Transaction `protobuf:"bytes,1,opt,name=txn" json:"txn"` // Version of distsqlrun protocol; a server accepts a certain range of // versions, up to its own version. See server.go for more details. - Version uint32 `protobuf:"varint,5,opt,name=version" json:"version"` - Flow FlowSpec `protobuf:"bytes,3,opt,name=flow" json:"flow"` - EvalContext EvalContext `protobuf:"bytes,6,opt,name=evalContext" json:"evalContext"` + Version DistSQLVersion `protobuf:"varint,5,opt,name=version,casttype=DistSQLVersion" json:"version"` + Flow FlowSpec `protobuf:"bytes,3,opt,name=flow" json:"flow"` + EvalContext EvalContext `protobuf:"bytes,6,opt,name=evalContext" json:"evalContext"` } func (m *SetupFlowRequest) Reset() { *m = SetupFlowRequest{} } @@ -779,7 +780,7 @@ func (m *SetupFlowRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Version |= (uint32(b) & 0x7F) << shift + m.Version |= (DistSQLVersion(b) & 0x7F) << shift if b < 0x80 { break } @@ -1397,45 +1398,46 @@ var ( func init() { proto.RegisterFile("cockroach/pkg/sql/distsqlrun/api.proto", fileDescriptorApi) } var fileDescriptorApi = []byte{ - // 631 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0x86, 0x33, 0x49, 0xfa, 0x77, 0xd2, 0x56, 0xf9, 0x46, 0xdf, 0xc2, 0x8a, 0x84, 0x1b, 0x59, - 0x50, 0x02, 0x12, 0x36, 0xaa, 0x80, 0x05, 0x62, 0xd5, 0x16, 0x90, 0x80, 0x42, 0x71, 0xba, 0x40, - 0x6c, 0xaa, 0xe9, 0x64, 0x48, 0xac, 0x4e, 0x66, 0x9c, 0x99, 0x71, 0x5b, 0xee, 0x82, 0x4b, 0xe0, - 0x02, 0xb8, 0x05, 0x16, 0xec, 0xb2, 0x64, 0xc9, 0x0a, 0x41, 0xb8, 0x0e, 0x24, 0x64, 0x67, 0xea, - 0xb8, 0x29, 0x89, 0xca, 0x6e, 0x7c, 0xe6, 0x79, 0xdf, 0x33, 0xe7, 0xc7, 0xb0, 0x49, 0x25, 0x3d, - 0x56, 0x92, 0xd0, 0x5e, 0x10, 0x1f, 0x77, 0x03, 0x3d, 0xe0, 0x41, 0x27, 0xd2, 0x46, 0x0f, 0xb8, - 0x4a, 0x44, 0x40, 0xe2, 0xc8, 0x8f, 0x95, 0x34, 0x12, 0x3b, 0x39, 0xe7, 0xeb, 0x01, 0xf7, 0x27, - 0x4c, 0xa3, 0x79, 0xd1, 0x21, 0x3b, 0xc5, 0x47, 0x41, 0x87, 0x18, 0x32, 0xd6, 0x36, 0x6e, 0xce, - 0xcd, 0x51, 0x00, 0xef, 0xcc, 0x05, 0x63, 0x25, 0x29, 0xd3, 0x5a, 0x2a, 0x6d, 0xf1, 0xa9, 0xb7, - 0x27, 0x26, 0xe2, 0x41, 0x8f, 0xd3, 0xc0, 0x44, 0x7d, 0xa6, 0x0d, 0xe9, 0xc7, 0x96, 0xfb, 0xbf, - 0x2b, 0xbb, 0x32, 0x3b, 0x06, 0xe9, 0x69, 0x1c, 0xf5, 0x7e, 0x23, 0xa8, 0xb7, 0x99, 0x49, 0xe2, - 0x27, 0x5c, 0x9e, 0x86, 0x6c, 0x90, 0x30, 0x6d, 0xf0, 0x03, 0xa8, 0x98, 0x33, 0xe1, 0xa0, 0x26, - 0x6a, 0xd5, 0xb6, 0x5c, 0x7f, 0x52, 0xb4, 0x2d, 0xcb, 0x3f, 0x50, 0x44, 0x68, 0x42, 0x4d, 0x24, - 0xc5, 0x76, 0x75, 0xf8, 0x7d, 0xa3, 0x14, 0xa6, 0x02, 0xfc, 0x08, 0xaa, 0xef, 0xb8, 0x3c, 0x75, - 0x2a, 0x99, 0xd0, 0xf3, 0x67, 0x75, 0xcb, 0x4f, 0x93, 0xb5, 0x63, 0x46, 0xad, 0x38, 0x53, 0x61, - 0x17, 0x96, 0x4e, 0x98, 0xd2, 0x91, 0x14, 0xce, 0x42, 0x13, 0xb5, 0xd6, 0xec, 0xe5, 0x79, 0x10, - 0xef, 0x41, 0x8d, 0x9d, 0x10, 0xbe, 0x23, 0x85, 0x61, 0x67, 0xc6, 0x59, 0xcc, 0x92, 0xdc, 0x98, - 0x9d, 0xe4, 0xf1, 0x04, 0xb6, 0x56, 0x45, 0xfd, 0xb3, 0xea, 0x72, 0xb9, 0x5e, 0xf1, 0x3e, 0x95, - 0xa1, 0x56, 0x00, 0xf1, 0x3d, 0xc0, 0xda, 0xf4, 0xcd, 0xc1, 0x79, 0xf3, 0x5e, 0x12, 0x21, 0x75, - 0xd6, 0x89, 0x8a, 0x35, 0xf9, 0xcb, 0x3d, 0xde, 0x82, 0xff, 0xcc, 0x99, 0x98, 0x12, 0x95, 0x0b, - 0xa2, 0xcb, 0xd7, 0xf8, 0x15, 0xd4, 0x29, 0x4f, 0xb4, 0x61, 0x2a, 0xbf, 0xb0, 0x8d, 0xbb, 0x56, - 0xa8, 0x29, 0x1d, 0xa7, 0xdf, 0xe3, 0xd4, 0xcf, 0x21, 0xeb, 0x78, 0x49, 0x8c, 0x9b, 0xb0, 0xcc, - 0x25, 0x25, 0xe9, 0x50, 0x9c, 0x6a, 0x13, 0xb5, 0x56, 0x2c, 0x99, 0x47, 0x53, 0x22, 0xdd, 0xb3, - 0x23, 0xa2, 0x59, 0xd6, 0xe2, 0x9c, 0x38, 0x8f, 0x62, 0x17, 0x40, 0x33, 0xa2, 0x68, 0x6f, 0x9f, - 0x98, 0x9e, 0xb3, 0xd8, 0xac, 0xb4, 0x56, 0xc2, 0x42, 0xc4, 0x7b, 0x0a, 0xeb, 0xed, 0xa8, 0x1f, - 0x73, 0x16, 0x32, 0x1d, 0x4b, 0xa1, 0x19, 0xbe, 0x0f, 0x0b, 0x4c, 0x29, 0xa9, 0xec, 0xb6, 0x6c, - 0xcc, 0x99, 0x47, 0x8a, 0x85, 0x63, 0xda, 0xfb, 0x8c, 0x60, 0x7d, 0x47, 0x0a, 0x9d, 0xf4, 0x99, - 0x6a, 0x47, 0x5d, 0x41, 0x38, 0x7e, 0x0e, 0x6b, 0x1d, 0x45, 0x22, 0x71, 0xa8, 0xc6, 0x6b, 0x68, - 0x1d, 0x37, 0x67, 0x3b, 0xee, 0xa6, 0xb8, 0x5d, 0xda, 0x70, 0xb5, 0x53, 0xf8, 0xc2, 0x6f, 0x00, - 0xeb, 0x74, 0xad, 0x0f, 0xd3, 0xd5, 0xca, 0x1d, 0xcb, 0x99, 0xe3, 0xed, 0xd9, 0x8e, 0xd3, 0xbf, - 0x42, 0x58, 0xd7, 0x53, 0x91, 0x87, 0xd5, 0xe1, 0xc7, 0x0d, 0xe4, 0xad, 0xc3, 0x6a, 0x31, 0xfb, - 0xd6, 0x97, 0x32, 0x2c, 0xed, 0x46, 0xda, 0xb4, 0x5f, 0xbf, 0xc0, 0x3d, 0xa8, 0x85, 0x89, 0x68, - 0xbf, 0x17, 0x34, 0xd5, 0xe1, 0xd6, 0xec, 0x74, 0x17, 0x3b, 0xd0, 0xb8, 0x35, 0x9b, 0xdc, 0x57, - 0xb2, 0x93, 0x50, 0xa6, 0xf6, 0x98, 0xd6, 0xa4, 0xcb, 0xbc, 0x52, 0x0b, 0xdd, 0x45, 0x98, 0xc2, - 0x4a, 0xfe, 0x62, 0xfc, 0x0f, 0x65, 0x35, 0xe6, 0xbc, 0xe9, 0xe2, 0x7c, 0xbd, 0x12, 0xee, 0x02, - 0x64, 0xff, 0xab, 0x51, 0x8c, 0xf4, 0xf1, 0xd5, 0xdf, 0xd8, 0xb8, 0x72, 0xe1, 0xe3, 0x6a, 0xb6, - 0xaf, 0x0f, 0x7f, 0xba, 0xa5, 0xe1, 0xc8, 0x45, 0x5f, 0x47, 0x2e, 0xfa, 0x36, 0x72, 0xd1, 0x8f, - 0x91, 0x8b, 0x3e, 0xfc, 0x72, 0x4b, 0x6f, 0x61, 0x22, 0xfd, 0x13, 0x00, 0x00, 0xff, 0xff, 0x42, - 0xc4, 0x83, 0x8f, 0xac, 0x05, 0x00, 0x00, + // 644 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xdf, 0x6e, 0xd3, 0x30, + 0x14, 0xc6, 0x9b, 0xb6, 0xfb, 0xe7, 0x6e, 0x55, 0xb1, 0x10, 0x8a, 0x2a, 0x91, 0x56, 0x11, 0x8c, + 0x82, 0x44, 0x32, 0x4d, 0xc0, 0x05, 0xe2, 0x6a, 0x1b, 0x20, 0x01, 0x83, 0x91, 0x4e, 0x08, 0x71, + 0x33, 0x79, 0xae, 0x69, 0xa3, 0xb9, 0x76, 0x6a, 0x3b, 0xdb, 0x78, 0x0a, 0x78, 0x04, 0x1e, 0x80, + 0x57, 0xe0, 0x82, 0xbb, 0x5e, 0x72, 0xc9, 0xd5, 0x04, 0xe5, 0x2d, 0xb8, 0x42, 0x4e, 0xbd, 0x34, + 0xed, 0x48, 0x35, 0xee, 0x9c, 0x73, 0x7e, 0xdf, 0x77, 0x7c, 0x8e, 0x4f, 0xc0, 0x3a, 0xe6, 0xf8, + 0x48, 0x70, 0x84, 0x7b, 0x7e, 0x74, 0xd4, 0xf5, 0xe5, 0x80, 0xfa, 0x9d, 0x50, 0x2a, 0x39, 0xa0, + 0x22, 0x66, 0x3e, 0x8a, 0x42, 0x2f, 0x12, 0x5c, 0x71, 0x68, 0xa7, 0x9c, 0x27, 0x07, 0xd4, 0x9b, + 0x30, 0xf5, 0xe6, 0xb4, 0x43, 0x72, 0x8a, 0x0e, 0xfd, 0x0e, 0x52, 0x68, 0xac, 0xad, 0xdf, 0x9a, + 0x5b, 0x23, 0x03, 0xde, 0x9d, 0x0b, 0x46, 0x82, 0x63, 0x22, 0x25, 0x17, 0xd2, 0xe0, 0x33, 0x77, + 0x8f, 0x55, 0x48, 0xfd, 0x1e, 0xc5, 0xbe, 0x0a, 0xfb, 0x44, 0x2a, 0xd4, 0x8f, 0x0c, 0x77, 0xb5, + 0xcb, 0xbb, 0x3c, 0x39, 0xfa, 0xfa, 0x34, 0x8e, 0xba, 0x1f, 0x8b, 0xa0, 0xd6, 0x26, 0x2a, 0x8e, + 0x9e, 0x50, 0x7e, 0x12, 0x90, 0x41, 0x4c, 0xa4, 0x82, 0x0f, 0x40, 0x49, 0x9d, 0x32, 0xdb, 0x6a, + 0x5a, 0xad, 0xca, 0xa6, 0xe3, 0x4d, 0x9a, 0x36, 0x6d, 0x79, 0xfb, 0x02, 0x31, 0x89, 0xb0, 0x0a, + 0x39, 0xdb, 0x2a, 0x0f, 0xcf, 0x1a, 0x85, 0x40, 0x0b, 0xe0, 0x23, 0x50, 0x7e, 0x4f, 0xf9, 0x89, + 0x5d, 0x4a, 0x84, 0xae, 0x97, 0x37, 0x2d, 0x4f, 0x17, 0x6b, 0x47, 0x04, 0x1b, 0x71, 0xa2, 0x82, + 0x1b, 0x60, 0xe9, 0x98, 0x08, 0x19, 0x72, 0x66, 0x2f, 0x34, 0xad, 0xd6, 0xda, 0xd6, 0x35, 0x9d, + 0xfc, 0x73, 0xd6, 0xa8, 0xee, 0x84, 0x52, 0xb5, 0x5f, 0xbf, 0x78, 0x33, 0xce, 0x06, 0xe7, 0x18, + 0xdc, 0x05, 0x15, 0x72, 0x8c, 0xe8, 0x36, 0x67, 0x8a, 0x9c, 0x2a, 0x7b, 0x31, 0x29, 0x7b, 0x33, + 0xbf, 0xec, 0xe3, 0x09, 0x6c, 0x2a, 0x67, 0xf5, 0xcf, 0xca, 0xcb, 0xc5, 0x5a, 0xc9, 0xfd, 0x52, + 0x04, 0x95, 0x0c, 0x08, 0xef, 0x01, 0x28, 0x55, 0x5f, 0xed, 0x9f, 0x8f, 0xf3, 0x25, 0x62, 0x5c, + 0x26, 0xb3, 0x29, 0x19, 0x93, 0x7f, 0xe4, 0xe1, 0x26, 0xb8, 0xa2, 0x4e, 0xd9, 0x8c, 0xa8, 0x98, + 0x11, 0x5d, 0x4c, 0xc3, 0x57, 0xa0, 0x86, 0x69, 0x2c, 0x15, 0x11, 0x69, 0xc2, 0x8c, 0xf2, 0x7a, + 0xa6, 0x27, 0xfd, 0xc0, 0x5e, 0x8f, 0x62, 0x2f, 0x85, 0x8c, 0xe3, 0x05, 0x31, 0x6c, 0x82, 0x65, + 0xca, 0x31, 0xd2, 0xcf, 0x64, 0x97, 0x9b, 0x56, 0x6b, 0xc5, 0x90, 0x69, 0x54, 0x13, 0x7a, 0xf3, + 0x0e, 0x91, 0x24, 0xc9, 0xd0, 0x53, 0xe2, 0x3c, 0x0a, 0x1d, 0x00, 0x24, 0x41, 0x02, 0xf7, 0xf6, + 0x90, 0xea, 0xd9, 0x8b, 0xcd, 0x52, 0x6b, 0x25, 0xc8, 0x44, 0xdc, 0xa7, 0xa0, 0xda, 0x0e, 0xfb, + 0x11, 0x25, 0x01, 0x91, 0x11, 0x67, 0x92, 0xc0, 0xfb, 0x60, 0x81, 0x08, 0xc1, 0x85, 0xd9, 0x9f, + 0xc6, 0x9c, 0xf7, 0xd0, 0x58, 0x30, 0xa6, 0xdd, 0xaf, 0x16, 0xa8, 0x6e, 0x73, 0x26, 0xe3, 0x3e, + 0x11, 0xed, 0xb0, 0xcb, 0x10, 0x85, 0xcf, 0xc1, 0x5a, 0x47, 0xa0, 0x90, 0x1d, 0x88, 0xf1, 0x62, + 0x1a, 0xc7, 0xf5, 0x7c, 0xc7, 0x1d, 0x8d, 0x9b, 0x35, 0x0e, 0x56, 0x3b, 0x99, 0x2f, 0xf8, 0x16, + 0x40, 0xa9, 0x17, 0xfd, 0x40, 0x2f, 0x5b, 0xea, 0x58, 0x4c, 0x1c, 0xef, 0xe4, 0x3b, 0xce, 0xfe, + 0x1c, 0x41, 0x4d, 0xce, 0x44, 0x1e, 0x96, 0x87, 0x9f, 0x1b, 0x96, 0x5b, 0x05, 0xab, 0xd9, 0xea, + 0x9b, 0xdf, 0x8a, 0x60, 0xc9, 0x2c, 0x2e, 0xec, 0x81, 0x4a, 0x10, 0xb3, 0xf6, 0x07, 0x86, 0xb5, + 0x0e, 0xb6, 0xf2, 0xcb, 0x4d, 0x4f, 0xa0, 0x7e, 0x3b, 0x9f, 0xdc, 0x13, 0xbc, 0x13, 0x63, 0x22, + 0x76, 0x89, 0x94, 0xa8, 0x4b, 0xdc, 0x42, 0xcb, 0xda, 0xb0, 0x20, 0x06, 0x2b, 0xe9, 0x8d, 0xe1, + 0x7f, 0xb4, 0x55, 0x9f, 0x73, 0xa7, 0xe9, 0xf7, 0x75, 0x0b, 0xb0, 0x0b, 0x40, 0xf2, 0x07, 0x2b, + 0x41, 0x50, 0x1f, 0x5e, 0xfe, 0x8e, 0xf5, 0x4b, 0x37, 0x3e, 0xee, 0x66, 0xeb, 0xc6, 0xf0, 0x97, + 0x53, 0x18, 0x8e, 0x1c, 0xeb, 0xfb, 0xc8, 0xb1, 0x7e, 0x8c, 0x1c, 0xeb, 0xe7, 0xc8, 0xb1, 0x3e, + 0xfd, 0x76, 0x0a, 0xef, 0xc0, 0x44, 0xfa, 0x37, 0x00, 0x00, 0xff, 0xff, 0xd0, 0xae, 0x2c, 0x54, + 0xbe, 0x05, 0x00, 0x00, } diff --git a/pkg/sql/distsqlrun/api.proto b/pkg/sql/distsqlrun/api.proto index fea48978bdf6..13c7d415cc74 100644 --- a/pkg/sql/distsqlrun/api.proto +++ b/pkg/sql/distsqlrun/api.proto @@ -31,7 +31,8 @@ message SetupFlowRequest { // Version of distsqlrun protocol; a server accepts a certain range of // versions, up to its own version. See server.go for more details. - optional uint32 version = 5 [(gogoproto.nullable) = false]; + optional uint32 version = 5 [(gogoproto.nullable) = false, + (gogoproto.casttype) = "DistSQLVersion"]; optional FlowSpec flow = 3 [(gogoproto.nullable) = false]; diff --git a/pkg/sql/distsqlrun/data.pb.go b/pkg/sql/distsqlrun/data.pb.go index 16795771f020..23717a209d8f 100644 --- a/pkg/sql/distsqlrun/data.pb.go +++ b/pkg/sql/distsqlrun/data.pb.go @@ -689,6 +689,22 @@ func (*RemoteProducerMetadata_TraceData) Descriptor() ([]byte, []int) { return fileDescriptorData, []int{10, 1} } +// DistSQLVersionGossipInfo represents the DistSQL server version information +// that gets gossiped for each node. This is used by planners to avoid planning +// on nodes with incompatible version during rolling cluster updates. +// +// For the meaning of the fields, see the corresponding constants in +// distsqlrun/server.go. +type DistSQLVersionGossipInfo struct { + Version DistSQLVersion `protobuf:"varint,1,opt,name=version,casttype=DistSQLVersion" json:"version"` + MinAcceptedVersion DistSQLVersion `protobuf:"varint,2,opt,name=min_accepted_version,json=minAcceptedVersion,casttype=DistSQLVersion" json:"min_accepted_version"` +} + +func (m *DistSQLVersionGossipInfo) Reset() { *m = DistSQLVersionGossipInfo{} } +func (m *DistSQLVersionGossipInfo) String() string { return proto.CompactTextString(m) } +func (*DistSQLVersionGossipInfo) ProtoMessage() {} +func (*DistSQLVersionGossipInfo) Descriptor() ([]byte, []int) { return fileDescriptorData, []int{11} } + func init() { proto.RegisterType((*Error)(nil), "cockroach.sql.distsqlrun.Error") proto.RegisterType((*Expression)(nil), "cockroach.sql.distsqlrun.Expression") @@ -707,6 +723,7 @@ func init() { proto.RegisterType((*RemoteProducerMetadata)(nil), "cockroach.sql.distsqlrun.RemoteProducerMetadata") proto.RegisterType((*RemoteProducerMetadata_RangeInfos)(nil), "cockroach.sql.distsqlrun.RemoteProducerMetadata.RangeInfos") proto.RegisterType((*RemoteProducerMetadata_TraceData)(nil), "cockroach.sql.distsqlrun.RemoteProducerMetadata.TraceData") + proto.RegisterType((*DistSQLVersionGossipInfo)(nil), "cockroach.sql.distsqlrun.DistSQLVersionGossipInfo") proto.RegisterEnum("cockroach.sql.distsqlrun.Ordering_Column_Direction", Ordering_Column_Direction_name, Ordering_Column_Direction_value) proto.RegisterEnum("cockroach.sql.distsqlrun.StreamEndpointSpec_Type", StreamEndpointSpec_Type_name, StreamEndpointSpec_Type_value) proto.RegisterEnum("cockroach.sql.distsqlrun.InputSyncSpec_Type", InputSyncSpec_Type_name, InputSyncSpec_Type_value) @@ -1347,6 +1364,30 @@ func (m *RemoteProducerMetadata_TraceData) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *DistSQLVersionGossipInfo) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DistSQLVersionGossipInfo) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0x8 + i++ + i = encodeVarintData(dAtA, i, uint64(m.Version)) + dAtA[i] = 0x10 + i++ + i = encodeVarintData(dAtA, i, uint64(m.MinAcceptedVersion)) + return i, nil +} + func encodeFixed64Data(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -1638,6 +1679,14 @@ func (m *RemoteProducerMetadata_TraceData) Size() (n int) { return n } +func (m *DistSQLVersionGossipInfo) Size() (n int) { + var l int + _ = l + n += 1 + sovData(uint64(m.Version)) + n += 1 + sovData(uint64(m.MinAcceptedVersion)) + return n +} + func sovData(x uint64) (n int) { for { n++ @@ -3625,6 +3674,94 @@ func (m *RemoteProducerMetadata_TraceData) Unmarshal(dAtA []byte) error { } return nil } +func (m *DistSQLVersionGossipInfo) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowData + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DistSQLVersionGossipInfo: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DistSQLVersionGossipInfo: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + m.Version = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowData + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Version |= (DistSQLVersion(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MinAcceptedVersion", wireType) + } + m.MinAcceptedVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowData + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MinAcceptedVersion |= (DistSQLVersion(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipData(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthData + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipData(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -3733,91 +3870,95 @@ var ( func init() { proto.RegisterFile("cockroach/pkg/sql/distsqlrun/data.proto", fileDescriptorData) } var fileDescriptorData = []byte{ - // 1371 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x4d, 0x6f, 0xdb, 0xc6, - 0x16, 0x15, 0xad, 0xef, 0x2b, 0x7f, 0x28, 0x83, 0x87, 0x40, 0xd0, 0xcb, 0x93, 0x13, 0xbe, 0xb4, - 0x4d, 0x83, 0x94, 0x4a, 0xdc, 0x45, 0x51, 0x67, 0x91, 0x48, 0x96, 0x1c, 0xa9, 0x4d, 0x2c, 0x83, - 0x72, 0x50, 0x24, 0x29, 0xc0, 0xd2, 0xe4, 0x58, 0x26, 0x42, 0x91, 0xf4, 0xcc, 0x30, 0xb2, 0x36, - 0xfd, 0x0d, 0x59, 0x76, 0x99, 0x7d, 0xb7, 0x01, 0xfa, 0x17, 0xbc, 0xec, 0x32, 0x68, 0x01, 0xa3, - 0x55, 0x7f, 0x43, 0x81, 0xa2, 0xab, 0x62, 0x86, 0x43, 0x7d, 0x58, 0x56, 0x13, 0x27, 0x3b, 0xce, - 0xcc, 0xb9, 0x67, 0xce, 0x3d, 0x73, 0xe7, 0x72, 0xe0, 0x13, 0xcb, 0xb7, 0x9e, 0x13, 0xdf, 0xb4, - 0x0e, 0xab, 0xc1, 0xf3, 0x5e, 0x95, 0x1e, 0xb9, 0x55, 0xdb, 0xa1, 0x8c, 0x1e, 0xb9, 0x24, 0xf4, - 0xaa, 0xb6, 0xc9, 0x4c, 0x2d, 0x20, 0x3e, 0xf3, 0x51, 0x69, 0x0c, 0xd4, 0xe8, 0x91, 0xab, 0x4d, - 0x40, 0xe5, 0xf5, 0x59, 0x0a, 0xf1, 0x15, 0xec, 0x57, 0xcd, 0xc0, 0x89, 0x42, 0xcb, 0x57, 0xcf, - 0x07, 0x4c, 0xc8, 0xcb, 0xea, 0xf9, 0x08, 0x4c, 0x88, 0x4f, 0xa8, 0xc4, 0x7c, 0x36, 0xaf, 0x34, - 0xe8, 0x0d, 0x1c, 0x82, 0xab, 0x41, 0x4f, 0x00, 0x67, 0xe1, 0x37, 0xe7, 0xe1, 0xf4, 0xc8, 0xdd, - 0x37, 0x29, 0xae, 0x52, 0x46, 0x42, 0x8b, 0x85, 0x04, 0xdb, 0x8b, 0xa9, 0x63, 0x2c, 0xf6, 0x2c, - 0xdf, 0xc6, 0xb6, 0x61, 0x9b, 0x2c, 0xec, 0x4b, 0xb8, 0x36, 0x0b, 0x0f, 0x99, 0xe3, 0x56, 0x19, - 0x31, 0x2d, 0xc7, 0xeb, 0x55, 0x09, 0xb6, 0x7c, 0xc2, 0x03, 0x68, 0x60, 0x7a, 0x12, 0xff, 0x9f, - 0x9e, 0xdf, 0xf3, 0xc5, 0x67, 0x95, 0x7f, 0x45, 0xb3, 0xea, 0x4f, 0x0a, 0xa4, 0x9b, 0x5c, 0x31, - 0xaa, 0x43, 0x2e, 0xe8, 0x19, 0x42, 0x7d, 0x49, 0xb9, 0xaa, 0xdc, 0x28, 0x6c, 0x94, 0x26, 0x5b, - 0x68, 0x32, 0x3b, 0x4d, 0x60, 0xeb, 0x85, 0xd1, 0xe9, 0x7a, 0x76, 0xf7, 0x81, 0x18, 0xb4, 0x12, - 0x7a, 0x36, 0xe8, 0x45, 0x1c, 0x4f, 0xe1, 0x12, 0xc1, 0x8c, 0x0c, 0xcd, 0x7d, 0x17, 0xef, 0x1d, - 0x7b, 0x62, 0xb2, 0xb4, 0x24, 0xc8, 0x6e, 0x4e, 0x91, 0x49, 0x67, 0xb5, 0xc7, 0xde, 0xa1, 0xe9, - 0xd9, 0x2e, 0xb6, 0xf5, 0x38, 0x28, 0x66, 0x9c, 0xa7, 0xd9, 0x4c, 0xfd, 0xf0, 0x6a, 0x3d, 0x51, - 0xcf, 0x41, 0xc6, 0xc6, 0xcc, 0x74, 0x5c, 0x75, 0x1b, 0xa0, 0x79, 0x1c, 0x10, 0x4c, 0xa9, 0xe3, - 0x7b, 0xa8, 0x02, 0xd9, 0x17, 0x98, 0xf0, 0x4f, 0x21, 0x3e, 0x5f, 0x4f, 0x9d, 0x9c, 0xae, 0x27, - 0xf4, 0x78, 0x12, 0x95, 0x20, 0x85, 0x8f, 0x83, 0x48, 0x4c, 0xbc, 0x28, 0x66, 0xd4, 0xbf, 0x14, - 0xc8, 0x75, 0x88, 0x8d, 0x89, 0xe3, 0xf5, 0x50, 0x1b, 0xb2, 0x96, 0xef, 0x86, 0x7d, 0x8f, 0x96, - 0x94, 0xab, 0xc9, 0x1b, 0x85, 0x8d, 0x4f, 0xb5, 0x45, 0x15, 0xa7, 0xc5, 0x41, 0xda, 0x96, 0x88, - 0x88, 0x77, 0x94, 0xf1, 0xe5, 0x57, 0x0a, 0x64, 0xa2, 0x15, 0xf4, 0x3f, 0xc1, 0x6a, 0x38, 0xf6, - 0xb1, 0x10, 0xb7, 0x22, 0xa1, 0x19, 0xcb, 0x77, 0xdb, 0xf6, 0x31, 0xfa, 0x06, 0xf2, 0xb6, 0x43, - 0xb0, 0xc5, 0xb8, 0x7a, 0x2e, 0x70, 0x75, 0xe3, 0xf3, 0x77, 0xde, 0x56, 0x6b, 0xc4, 0xa1, 0x92, - 0x75, 0xc2, 0xa5, 0x56, 0x20, 0x3f, 0x5e, 0x45, 0x59, 0x48, 0xd6, 0xba, 0x5b, 0xc5, 0x04, 0xca, - 0x41, 0xaa, 0xd1, 0xec, 0x6e, 0x15, 0x15, 0xf5, 0x4f, 0x05, 0x50, 0x97, 0x11, 0x6c, 0xf6, 0x9b, - 0x9e, 0x1d, 0xf8, 0x8e, 0xc7, 0xba, 0x01, 0xb6, 0xd0, 0xd7, 0x90, 0x62, 0xc3, 0x00, 0x0b, 0xad, - 0xab, 0x1b, 0x77, 0x16, 0x4b, 0x99, 0x8f, 0xd5, 0xf6, 0x86, 0x01, 0x8e, 0xed, 0xe5, 0x24, 0xe8, - 0x4b, 0xc8, 0x53, 0x01, 0x33, 0x1c, 0x5b, 0x24, 0x97, 0xae, 0x5f, 0xe1, 0xcb, 0xa3, 0xd3, 0xf5, - 0x5c, 0x14, 0xdf, 0x6e, 0xfc, 0x3d, 0xf5, 0xad, 0xe7, 0x22, 0x78, 0xdb, 0x46, 0x1f, 0x41, 0x81, - 0x99, 0xa4, 0x87, 0x99, 0x61, 0xda, 0x36, 0x29, 0x25, 0xa7, 0x8e, 0x0e, 0xa2, 0x85, 0x9a, 0x6d, - 0x13, 0xf5, 0x36, 0xa4, 0xf8, 0xae, 0x28, 0x0f, 0xe9, 0x87, 0x9d, 0xad, 0xda, 0xc3, 0x62, 0x02, - 0x01, 0x64, 0xf4, 0xe6, 0xa3, 0xce, 0x5e, 0xb3, 0xa8, 0xa0, 0x4b, 0xb0, 0xd2, 0x7d, 0xb2, 0xb3, - 0x65, 0xe8, 0xcd, 0xee, 0x6e, 0x67, 0xa7, 0xdb, 0x2c, 0x2e, 0xa9, 0xbf, 0x2e, 0xc1, 0x4a, 0xdb, - 0x0b, 0x42, 0xd6, 0x1d, 0x7a, 0x96, 0x48, 0x79, 0x7b, 0x26, 0xe5, 0x5b, 0x8b, 0x53, 0x9e, 0x09, - 0x9b, 0xcf, 0xb6, 0x01, 0x39, 0x5f, 0x9e, 0x8f, 0xac, 0x7b, 0xf5, 0xed, 0x27, 0x29, 0x19, 0xc6, - 0x91, 0xe8, 0x21, 0x64, 0x23, 0x13, 0x68, 0x29, 0x29, 0xaa, 0xf0, 0xd6, 0x45, 0xce, 0x20, 0x2e, - 0x44, 0x49, 0x81, 0xbe, 0x82, 0xe5, 0xa8, 0x26, 0x0d, 0x2e, 0x91, 0x96, 0x52, 0x82, 0xf2, 0xda, - 0x19, 0x4a, 0xd9, 0x6a, 0x64, 0x55, 0x4d, 0x25, 0x56, 0xb0, 0xc6, 0x33, 0x54, 0x55, 0xa5, 0xd7, - 0x2b, 0x90, 0x7f, 0xbc, 0xd3, 0xd1, 0x1b, 0x4d, 0xbd, 0xd9, 0x28, 0x26, 0x50, 0x01, 0xb2, 0xf1, - 0x40, 0x51, 0x5f, 0x67, 0xa0, 0xd8, 0x09, 0x59, 0x10, 0x32, 0xdd, 0x0f, 0x19, 0x26, 0xc2, 0xe0, - 0xf6, 0x8c, 0xc1, 0xd5, 0x7f, 0x31, 0xe5, 0x4c, 0xe4, 0xbc, 0xc7, 0x53, 0xee, 0x2c, 0x7d, 0xb8, - 0x3b, 0xd7, 0x60, 0xf9, 0xd0, 0xa4, 0x87, 0x46, 0x7c, 0xed, 0xb9, 0xe1, 0x2b, 0x7a, 0x81, 0xcf, - 0x45, 0x56, 0x50, 0xe4, 0xc2, 0x25, 0x62, 0x7a, 0x3d, 0x6c, 0x10, 0xa1, 0xca, 0xa0, 0x01, 0xb6, - 0x4a, 0x29, 0x71, 0xba, 0x9b, 0x17, 0x48, 0x44, 0xe7, 0x1c, 0x93, 0xb1, 0x14, 0xb2, 0x46, 0x66, - 0xa7, 0xcb, 0xaf, 0x93, 0xb0, 0x76, 0x06, 0x8a, 0x9e, 0x41, 0x9a, 0x77, 0xf2, 0xb8, 0x29, 0xdd, - 0x7b, 0xff, 0x5d, 0xb5, 0x6e, 0x60, 0xc6, 0x9d, 0x22, 0xe2, 0xe4, 0x0e, 0xd8, 0xf8, 0xc0, 0x0c, - 0x5d, 0x66, 0xd8, 0x98, 0xb2, 0xe8, 0x92, 0xea, 0x05, 0x39, 0xd7, 0xc0, 0x94, 0xa1, 0x3e, 0xe4, - 0xc5, 0x2f, 0xc8, 0xf1, 0x7a, 0x71, 0x49, 0xb6, 0x3f, 0x40, 0x43, 0x64, 0x6c, 0x53, 0x32, 0xc6, - 0x7d, 0x6b, 0xbc, 0x43, 0xf9, 0x05, 0xac, 0xce, 0x42, 0xd0, 0x15, 0xc8, 0x44, 0x07, 0x34, 0xd7, - 0x40, 0x79, 0x7f, 0xdd, 0x86, 0x5c, 0x1c, 0x2c, 0xfb, 0xe7, 0xf5, 0x05, 0xd5, 0xdd, 0xe0, 0x3f, - 0xd0, 0x33, 0x1b, 0x8f, 0x63, 0xcb, 0x9b, 0x90, 0xe2, 0xf6, 0xa0, 0x32, 0xa4, 0x29, 0x33, 0x09, - 0x13, 0x9b, 0x2d, 0x8f, 0xdd, 0xe2, 0x53, 0xe8, 0x32, 0x24, 0xb1, 0x17, 0x75, 0xb2, 0x68, 0x45, - 0xd1, 0xf9, 0x84, 0x7a, 0x4f, 0xde, 0x8c, 0x22, 0x2c, 0xef, 0xd6, 0xba, 0x5d, 0x63, 0xaf, 0xa5, - 0x77, 0x1e, 0x3f, 0x68, 0x45, 0xcd, 0xe8, 0x51, 0x5b, 0xd7, 0x3b, 0x7a, 0x51, 0xe1, 0x17, 0xa5, - 0xfe, 0xc4, 0x68, 0xd5, 0xba, 0xad, 0xe2, 0x12, 0x5a, 0x86, 0x5c, 0xfd, 0x89, 0xa1, 0xd7, 0x76, - 0x1e, 0x34, 0x8b, 0x49, 0xf5, 0xa5, 0x02, 0x79, 0x21, 0xaf, 0xed, 0x1d, 0xf8, 0x33, 0x29, 0x29, - 0xef, 0x9f, 0x12, 0xba, 0x2b, 0xef, 0x5d, 0xd4, 0x8c, 0xde, 0xf9, 0xd2, 0x8b, 0x20, 0xf5, 0x7b, - 0x58, 0xdd, 0x25, 0xbe, 0x1d, 0x5a, 0x98, 0xb4, 0xb0, 0x69, 0x63, 0x82, 0xee, 0x40, 0xf6, 0xc0, - 0xf5, 0x07, 0xbc, 0x97, 0x47, 0xde, 0x94, 0x38, 0xfc, 0x97, 0xd3, 0xf5, 0xcc, 0xb6, 0xeb, 0x0f, - 0xda, 0x8d, 0xd1, 0xf8, 0x4b, 0xcf, 0x70, 0x60, 0xdb, 0xfe, 0x80, 0x1f, 0x80, 0xfa, 0xa3, 0x02, - 0xcb, 0xb1, 0x80, 0x86, 0xc9, 0x4c, 0xf4, 0x5f, 0xc8, 0x13, 0x73, 0x60, 0xec, 0x0f, 0x19, 0xa6, - 0x91, 0x00, 0x3d, 0x47, 0xcc, 0x41, 0x9d, 0x8f, 0x91, 0x0e, 0xb9, 0x3e, 0x66, 0x26, 0x7f, 0xd0, - 0xc9, 0xc6, 0x70, 0x7b, 0x71, 0x8d, 0xea, 0xb8, 0xef, 0x33, 0x1c, 0x93, 0x3f, 0x92, 0x71, 0xb1, - 0x7d, 0x31, 0x0f, 0xba, 0x09, 0xab, 0x5e, 0xd8, 0x37, 0x70, 0x3f, 0x60, 0x43, 0x83, 0xf8, 0x03, - 0x2a, 0xfe, 0x42, 0x69, 0x89, 0x5b, 0xf6, 0xc2, 0x7e, 0x93, 0x2f, 0xe9, 0xfe, 0x80, 0xaa, 0x6f, - 0x14, 0x58, 0x9b, 0x10, 0x52, 0x6a, 0xf6, 0x30, 0xba, 0x0f, 0x99, 0x43, 0xe1, 0x9c, 0x7c, 0x52, - 0xdd, 0x58, 0xac, 0x68, 0xd6, 0x69, 0x5d, 0xc6, 0xa1, 0x1a, 0x64, 0xd8, 0x30, 0x88, 0x2a, 0x9b, - 0xe7, 0xf4, 0xff, 0xc5, 0x0c, 0xe3, 0xea, 0x89, 0xaf, 0x47, 0x14, 0x88, 0xee, 0x43, 0x4a, 0x98, - 0x92, 0x14, 0x12, 0x3e, 0x7e, 0xbb, 0x84, 0xc6, 0xc4, 0x0a, 0x11, 0xa9, 0x9e, 0x24, 0xe1, 0xf2, - 0xf9, 0x8e, 0xa1, 0x6f, 0x01, 0xa2, 0xe6, 0xe8, 0x78, 0x07, 0xbe, 0xcc, 0xf2, 0xee, 0x45, 0x7d, - 0x8f, 0x3a, 0x04, 0x97, 0x4e, 0x5b, 0x09, 0x3d, 0x4f, 0xe2, 0x11, 0xfa, 0x02, 0xd2, 0x78, 0xea, - 0x11, 0xb9, 0xbe, 0x98, 0x38, 0x7e, 0x39, 0x46, 0x78, 0xf4, 0x0c, 0x80, 0x3f, 0x86, 0xb1, 0x31, - 0x95, 0xf9, 0xe6, 0x85, 0x65, 0xed, 0x71, 0x0a, 0xee, 0x06, 0x57, 0xc5, 0xe2, 0x41, 0xb9, 0x03, - 0x30, 0x11, 0x8c, 0x6a, 0x67, 0x1c, 0xe0, 0xa7, 0x74, 0xe5, 0x9c, 0xd7, 0xee, 0x38, 0x24, 0x6e, - 0x78, 0xe3, 0x34, 0xcb, 0xdf, 0x41, 0x7e, 0xbc, 0x15, 0xea, 0xc2, 0x9a, 0xe5, 0xbb, 0x2e, 0xb6, - 0x98, 0x7c, 0xc0, 0xc7, 0x6d, 0x7f, 0xba, 0x03, 0xf0, 0xe7, 0xbe, 0x26, 0x9f, 0xfb, 0x9a, 0x2e, - 0x9f, 0xfb, 0x53, 0xbd, 0x7d, 0x75, 0x4c, 0xc1, 0x27, 0x69, 0x3d, 0x0b, 0xe9, 0x17, 0xa6, 0x1b, - 0xe2, 0xfa, 0xf5, 0x93, 0xdf, 0x2b, 0x89, 0x93, 0x51, 0x45, 0xf9, 0x79, 0x54, 0x51, 0xde, 0x8c, - 0x2a, 0xca, 0x6f, 0xa3, 0x8a, 0xf2, 0xf2, 0x8f, 0x4a, 0xe2, 0x29, 0x4c, 0xfc, 0xf8, 0x27, 0x00, - 0x00, 0xff, 0xff, 0x57, 0x82, 0xbf, 0xf7, 0x91, 0x0d, 0x00, 0x00, + // 1432 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x4d, 0x6f, 0xdb, 0x36, + 0x18, 0xb6, 0xe2, 0xef, 0xd7, 0xf9, 0x70, 0x89, 0xa2, 0x30, 0xbc, 0xce, 0x6e, 0xb5, 0x6e, 0xeb, + 0x8a, 0x4e, 0x6e, 0xb3, 0xc3, 0xb0, 0xf4, 0xd0, 0xda, 0xb1, 0x13, 0x7b, 0x4b, 0xe3, 0x4c, 0x4e, + 0x37, 0xb4, 0x1d, 0xa0, 0x29, 0x12, 0xe3, 0x08, 0x95, 0x25, 0x85, 0xa4, 0x9a, 0xe4, 0xb2, 0xdf, + 0xd0, 0xd3, 0xb0, 0x63, 0xef, 0xbb, 0x16, 0xd8, 0x5f, 0xc8, 0x71, 0xc7, 0x62, 0x03, 0x82, 0x2d, + 0xfb, 0x0d, 0x03, 0x86, 0x9e, 0x06, 0x52, 0x94, 0x3f, 0x92, 0xb8, 0x6d, 0xda, 0x9b, 0x48, 0xbe, + 0xcf, 0xc3, 0xe7, 0x7d, 0x48, 0xbe, 0x7a, 0xe1, 0x53, 0xcb, 0xb7, 0x9e, 0x10, 0xdf, 0xb4, 0x76, + 0x6a, 0xc1, 0x93, 0x7e, 0x8d, 0xee, 0xba, 0x35, 0xdb, 0xa1, 0x8c, 0xee, 0xba, 0x24, 0xf4, 0x6a, + 0xb6, 0xc9, 0x4c, 0x2d, 0x20, 0x3e, 0xf3, 0x51, 0x69, 0x18, 0xa8, 0xd1, 0x5d, 0x57, 0x1b, 0x05, + 0x95, 0xab, 0x93, 0x14, 0xe2, 0x2b, 0xd8, 0xaa, 0x99, 0x81, 0x13, 0x41, 0xcb, 0x57, 0xce, 0x0e, + 0x18, 0x91, 0x97, 0xd5, 0xb3, 0x23, 0x30, 0x21, 0x3e, 0xa1, 0x32, 0xe6, 0xf3, 0xd3, 0x4a, 0x83, + 0xfe, 0x9e, 0x43, 0x70, 0x2d, 0xe8, 0x8b, 0xc0, 0xc9, 0xf0, 0x1b, 0xa7, 0xc3, 0xe9, 0xae, 0xbb, + 0x65, 0x52, 0x5c, 0xa3, 0x8c, 0x84, 0x16, 0x0b, 0x09, 0xb6, 0xa7, 0x53, 0xc7, 0xb1, 0xd8, 0xb3, + 0x7c, 0x1b, 0xdb, 0x86, 0x6d, 0xb2, 0x70, 0x20, 0xc3, 0xb5, 0xc9, 0xf0, 0x90, 0x39, 0x6e, 0x8d, + 0x11, 0xd3, 0x72, 0xbc, 0x7e, 0x8d, 0x60, 0xcb, 0x27, 0x1c, 0x40, 0x03, 0xd3, 0x93, 0xf1, 0x17, + 0xfb, 0x7e, 0xdf, 0x17, 0x9f, 0x35, 0xfe, 0x15, 0xcd, 0xaa, 0xbf, 0x29, 0x90, 0x6e, 0x71, 0xc5, + 0xa8, 0x01, 0xb9, 0xa0, 0x6f, 0x08, 0xf5, 0x25, 0xe5, 0x8a, 0x72, 0xbd, 0xb0, 0x58, 0x1a, 0x6d, + 0xa1, 0xc9, 0xec, 0x34, 0x11, 0xdb, 0x28, 0x1c, 0x1f, 0x55, 0xb3, 0x1b, 0xab, 0x62, 0xd0, 0x4e, + 0xe8, 0xd9, 0xa0, 0x1f, 0x71, 0x3c, 0x82, 0x0b, 0x04, 0x33, 0x72, 0x60, 0x6e, 0xb9, 0x78, 0x73, + 0xdf, 0x13, 0x93, 0xa5, 0x19, 0x41, 0x76, 0x63, 0x8c, 0x4c, 0x3a, 0xab, 0x3d, 0xf0, 0x76, 0x4c, + 0xcf, 0x76, 0xb1, 0xad, 0xc7, 0xa0, 0x98, 0xf1, 0x34, 0xcd, 0x52, 0xea, 0x97, 0xe7, 0xd5, 0x44, + 0x23, 0x07, 0x19, 0x1b, 0x33, 0xd3, 0x71, 0xd5, 0x15, 0x80, 0xd6, 0x7e, 0x40, 0x30, 0xa5, 0x8e, + 0xef, 0xa1, 0x0a, 0x64, 0x9f, 0x62, 0xc2, 0x3f, 0x85, 0xf8, 0x7c, 0x23, 0x75, 0x78, 0x54, 0x4d, + 0xe8, 0xf1, 0x24, 0x2a, 0x41, 0x0a, 0xef, 0x07, 0x91, 0x98, 0x78, 0x51, 0xcc, 0xa8, 0xff, 0x29, + 0x90, 0xeb, 0x12, 0x1b, 0x13, 0xc7, 0xeb, 0xa3, 0x0e, 0x64, 0x2d, 0xdf, 0x0d, 0x07, 0x1e, 0x2d, + 0x29, 0x57, 0x92, 0xd7, 0x0b, 0x8b, 0x9f, 0x69, 0xd3, 0x6e, 0x9c, 0x16, 0x83, 0xb4, 0x65, 0x81, + 0x88, 0x77, 0x94, 0xf8, 0xf2, 0x73, 0x05, 0x32, 0xd1, 0x0a, 0xfa, 0x50, 0xb0, 0x1a, 0x8e, 0xbd, + 0x2f, 0xc4, 0xcd, 0xc9, 0xd0, 0x8c, 0xe5, 0xbb, 0x1d, 0x7b, 0x1f, 0x7d, 0x0f, 0x79, 0xdb, 0x21, + 0xd8, 0x62, 0x5c, 0x3d, 0x17, 0x38, 0xbf, 0xf8, 0xc5, 0x5b, 0x6f, 0xab, 0x35, 0x63, 0xa8, 0x64, + 0x1d, 0x71, 0xa9, 0x15, 0xc8, 0x0f, 0x57, 0x51, 0x16, 0x92, 0xf5, 0xde, 0x72, 0x31, 0x81, 0x72, + 0x90, 0x6a, 0xb6, 0x7a, 0xcb, 0x45, 0x45, 0xfd, 0x57, 0x01, 0xd4, 0x63, 0x04, 0x9b, 0x83, 0x96, + 0x67, 0x07, 0xbe, 0xe3, 0xb1, 0x5e, 0x80, 0x2d, 0xf4, 0x0d, 0xa4, 0xd8, 0x41, 0x80, 0x85, 0xd6, + 0xf9, 0xc5, 0xdb, 0xd3, 0xa5, 0x9c, 0xc6, 0x6a, 0x9b, 0x07, 0x01, 0x8e, 0xed, 0xe5, 0x24, 0xe8, + 0x2b, 0xc8, 0x53, 0x11, 0x66, 0x38, 0xb6, 0x48, 0x2e, 0xdd, 0xb8, 0xcc, 0x97, 0x8f, 0x8f, 0xaa, + 0xb9, 0x08, 0xdf, 0x69, 0xbe, 0x1a, 0xfb, 0xd6, 0x73, 0x51, 0x78, 0xc7, 0x46, 0x1f, 0x43, 0x81, + 0x99, 0xa4, 0x8f, 0x99, 0x61, 0xda, 0x36, 0x29, 0x25, 0xc7, 0x8e, 0x0e, 0xa2, 0x85, 0xba, 0x6d, + 0x13, 0xf5, 0x16, 0xa4, 0xf8, 0xae, 0x28, 0x0f, 0xe9, 0xb5, 0xee, 0x72, 0x7d, 0xad, 0x98, 0x40, + 0x00, 0x19, 0xbd, 0x75, 0xbf, 0xbb, 0xd9, 0x2a, 0x2a, 0xe8, 0x02, 0xcc, 0xf5, 0x1e, 0xae, 0x2f, + 0x1b, 0x7a, 0xab, 0xb7, 0xd1, 0x5d, 0xef, 0xb5, 0x8a, 0x33, 0xea, 0x9f, 0x33, 0x30, 0xd7, 0xf1, + 0x82, 0x90, 0xf5, 0x0e, 0x3c, 0x4b, 0xa4, 0xbc, 0x32, 0x91, 0xf2, 0xcd, 0xe9, 0x29, 0x4f, 0xc0, + 0x4e, 0x67, 0xdb, 0x84, 0x9c, 0x2f, 0xcf, 0x47, 0xde, 0x7b, 0xf5, 0xcd, 0x27, 0x29, 0x19, 0x86, + 0x48, 0xb4, 0x06, 0xd9, 0xc8, 0x04, 0x5a, 0x4a, 0x8a, 0x5b, 0x78, 0xf3, 0x3c, 0x67, 0x10, 0x5f, + 0x44, 0x49, 0x81, 0xbe, 0x86, 0xd9, 0xe8, 0x4e, 0x1a, 0x5c, 0x22, 0x2d, 0xa5, 0x04, 0xe5, 0xd5, + 0x13, 0x94, 0xb2, 0xd4, 0xc8, 0x5b, 0x35, 0x96, 0x58, 0xc1, 0x1a, 0xce, 0x50, 0x55, 0x95, 0x5e, + 0xcf, 0x41, 0xfe, 0xc1, 0x7a, 0x57, 0x6f, 0xb6, 0xf4, 0x56, 0xb3, 0x98, 0x40, 0x05, 0xc8, 0xc6, + 0x03, 0x45, 0x7d, 0x91, 0x81, 0x62, 0x37, 0x64, 0x41, 0xc8, 0x74, 0x3f, 0x64, 0x98, 0x08, 0x83, + 0x3b, 0x13, 0x06, 0xd7, 0x5e, 0x63, 0xca, 0x09, 0xe4, 0x69, 0x8f, 0xc7, 0xdc, 0x99, 0x79, 0x7f, + 0x77, 0xae, 0xc2, 0xec, 0x8e, 0x49, 0x77, 0x8c, 0xf8, 0xd9, 0x73, 0xc3, 0xe7, 0xf4, 0x02, 0x9f, + 0x8b, 0xac, 0xa0, 0xc8, 0x85, 0x0b, 0xc4, 0xf4, 0xfa, 0xd8, 0x20, 0x42, 0x95, 0x41, 0x03, 0x6c, + 0x95, 0x52, 0xe2, 0x74, 0x97, 0xce, 0x91, 0x88, 0xce, 0x39, 0x46, 0x63, 0x29, 0x64, 0x81, 0x4c, + 0x4e, 0x97, 0x5f, 0x24, 0x61, 0xe1, 0x44, 0x28, 0x7a, 0x0c, 0x69, 0x5e, 0xc9, 0xe3, 0xa2, 0x74, + 0xf7, 0xdd, 0x77, 0xd5, 0x7a, 0x81, 0x19, 0x57, 0x8a, 0x88, 0x93, 0x3b, 0x60, 0xe3, 0x6d, 0x33, + 0x74, 0x99, 0x61, 0x63, 0xca, 0xa2, 0x47, 0xaa, 0x17, 0xe4, 0x5c, 0x13, 0x53, 0x86, 0x06, 0x90, + 0x17, 0xbf, 0x20, 0xc7, 0xeb, 0xc7, 0x57, 0xb2, 0xf3, 0x1e, 0x1a, 0x22, 0x63, 0x5b, 0x92, 0x31, + 0xae, 0x5b, 0xc3, 0x1d, 0xca, 0x4f, 0x61, 0x7e, 0x32, 0x04, 0x5d, 0x86, 0x4c, 0x74, 0x40, 0xa7, + 0x0a, 0x28, 0xaf, 0xaf, 0x2b, 0x90, 0x8b, 0xc1, 0xb2, 0x7e, 0x5e, 0x9b, 0x72, 0xbb, 0x9b, 0xfc, + 0x07, 0x7a, 0x62, 0xe3, 0x21, 0xb6, 0xbc, 0x04, 0x29, 0x6e, 0x0f, 0x2a, 0x43, 0x9a, 0x32, 0x93, + 0x30, 0xb1, 0xd9, 0xec, 0xd0, 0x2d, 0x3e, 0x85, 0x2e, 0x41, 0x12, 0x7b, 0x51, 0x25, 0x8b, 0x56, + 0x14, 0x9d, 0x4f, 0xa8, 0x77, 0xe5, 0xcb, 0x28, 0xc2, 0xec, 0x46, 0xbd, 0xd7, 0x33, 0x36, 0xdb, + 0x7a, 0xf7, 0xc1, 0x6a, 0x3b, 0x2a, 0x46, 0xf7, 0x3b, 0xba, 0xde, 0xd5, 0x8b, 0x0a, 0x7f, 0x28, + 0x8d, 0x87, 0x46, 0xbb, 0xde, 0x6b, 0x17, 0x67, 0xd0, 0x2c, 0xe4, 0x1a, 0x0f, 0x0d, 0xbd, 0xbe, + 0xbe, 0xda, 0x2a, 0x26, 0xd5, 0x67, 0x0a, 0xe4, 0x85, 0xbc, 0x8e, 0xb7, 0xed, 0x4f, 0xa4, 0xa4, + 0xbc, 0x7b, 0x4a, 0xe8, 0x8e, 0x7c, 0x77, 0x51, 0x31, 0x7a, 0xeb, 0x47, 0x2f, 0x40, 0xea, 0x4f, + 0x30, 0xbf, 0x41, 0x7c, 0x3b, 0xb4, 0x30, 0x69, 0x63, 0xd3, 0xc6, 0x04, 0xdd, 0x86, 0xec, 0xb6, + 0xeb, 0xef, 0xf1, 0x5a, 0x1e, 0x79, 0x53, 0xe2, 0xe1, 0x7f, 0x1c, 0x55, 0x33, 0x2b, 0xae, 0xbf, + 0xd7, 0x69, 0x1e, 0x0f, 0xbf, 0xf4, 0x0c, 0x0f, 0xec, 0xd8, 0xef, 0xf1, 0x03, 0x50, 0x7f, 0x55, + 0x60, 0x36, 0x16, 0xd0, 0x34, 0x99, 0x89, 0x3e, 0x80, 0x3c, 0x31, 0xf7, 0x8c, 0xad, 0x03, 0x86, + 0x69, 0x24, 0x40, 0xcf, 0x11, 0x73, 0xaf, 0xc1, 0xc7, 0x48, 0x87, 0xdc, 0x00, 0x33, 0x93, 0x37, + 0x74, 0xb2, 0x30, 0xdc, 0x9a, 0x7e, 0x47, 0x75, 0x3c, 0xf0, 0x19, 0x8e, 0xc9, 0xef, 0x4b, 0x5c, + 0x6c, 0x5f, 0xcc, 0x83, 0x6e, 0xc0, 0xbc, 0x17, 0x0e, 0x0c, 0x3c, 0x08, 0xd8, 0x81, 0x41, 0xfc, + 0x3d, 0x2a, 0xfe, 0x42, 0x69, 0x19, 0x37, 0xeb, 0x85, 0x83, 0x16, 0x5f, 0xd2, 0xfd, 0x3d, 0xaa, + 0xbe, 0x54, 0x60, 0x61, 0x44, 0x48, 0xa9, 0xd9, 0xc7, 0xe8, 0x1e, 0x64, 0x76, 0x84, 0x73, 0xb2, + 0xa5, 0xba, 0x3e, 0x5d, 0xd1, 0xa4, 0xd3, 0xba, 0xc4, 0xa1, 0x3a, 0x64, 0xd8, 0x41, 0x10, 0xdd, + 0x6c, 0x9e, 0xd3, 0x47, 0xd3, 0x19, 0x86, 0xb7, 0x27, 0x7e, 0x1e, 0x11, 0x10, 0xdd, 0x83, 0x94, + 0x30, 0x25, 0x29, 0x24, 0x7c, 0xf2, 0x66, 0x09, 0xcd, 0x91, 0x15, 0x02, 0xa9, 0x1e, 0x26, 0xe1, + 0xd2, 0xd9, 0x8e, 0xa1, 0x1f, 0x00, 0xa2, 0xe2, 0xe8, 0x78, 0xdb, 0xbe, 0xcc, 0xf2, 0xce, 0x79, + 0x7d, 0x8f, 0x2a, 0x04, 0x97, 0x4e, 0xdb, 0x09, 0x3d, 0x4f, 0xe2, 0x11, 0xfa, 0x12, 0xd2, 0x78, + 0xac, 0x89, 0xac, 0x4e, 0x27, 0x8e, 0x3b, 0xc7, 0x28, 0x1e, 0x3d, 0x06, 0xe0, 0xcd, 0x30, 0x36, + 0xc6, 0x32, 0x5f, 0x3a, 0xb7, 0xac, 0x4d, 0x4e, 0xc1, 0xdd, 0xe0, 0xaa, 0x58, 0x3c, 0x28, 0x77, + 0x01, 0x46, 0x82, 0x51, 0xfd, 0x84, 0x03, 0xfc, 0x94, 0x2e, 0x9f, 0xd1, 0xed, 0x0e, 0x21, 0x71, + 0xc1, 0x1b, 0xa6, 0x59, 0xfe, 0x11, 0xf2, 0xc3, 0xad, 0x50, 0x0f, 0x16, 0x2c, 0xdf, 0x75, 0xb1, + 0xc5, 0x64, 0x03, 0x1f, 0x97, 0xfd, 0xf1, 0x0a, 0xc0, 0xdb, 0x7d, 0x4d, 0xb6, 0xfb, 0x9a, 0x2e, + 0xdb, 0xfd, 0xb1, 0xda, 0x3e, 0x3f, 0xa4, 0xe0, 0x93, 0xb4, 0x91, 0x85, 0xf4, 0x53, 0xd3, 0x0d, + 0xb1, 0xfa, 0xb3, 0x02, 0xa5, 0xa6, 0x43, 0x59, 0xef, 0xdb, 0xb5, 0xef, 0xa2, 0xde, 0x78, 0xd5, + 0xa7, 0xd4, 0x09, 0x84, 0xdd, 0xb7, 0x26, 0xbb, 0xe8, 0xb9, 0xc6, 0x25, 0x4e, 0xf6, 0xea, 0xa8, + 0x3a, 0x3f, 0x09, 0x19, 0xf5, 0xd5, 0x6d, 0xb8, 0x38, 0x70, 0x3c, 0xc3, 0xb4, 0x2c, 0x1c, 0x70, + 0xbd, 0x31, 0x7c, 0xe6, 0xb5, 0x70, 0x34, 0x70, 0xbc, 0xba, 0x84, 0xc8, 0xb9, 0xc6, 0xb5, 0xc3, + 0xbf, 0x2b, 0x89, 0xc3, 0xe3, 0x8a, 0xf2, 0xfb, 0x71, 0x45, 0x79, 0x79, 0x5c, 0x51, 0xfe, 0x3a, + 0xae, 0x28, 0xcf, 0xfe, 0xa9, 0x24, 0x1e, 0xc1, 0xe8, 0xa0, 0xfe, 0x0f, 0x00, 0x00, 0xff, 0xff, + 0xa3, 0x33, 0x0f, 0xa3, 0x2a, 0x0e, 0x00, 0x00, } diff --git a/pkg/sql/distsqlrun/data.proto b/pkg/sql/distsqlrun/data.proto index 6456fab4a7d5..37e478ec9385 100644 --- a/pkg/sql/distsqlrun/data.proto +++ b/pkg/sql/distsqlrun/data.proto @@ -233,3 +233,17 @@ message RemoteProducerMetadata { TraceData trace_data = 3; } } + +// DistSQLVersionGossipInfo represents the DistSQL server version information +// that gets gossiped for each node. This is used by planners to avoid planning +// on nodes with incompatible version during rolling cluster updates. +// +// For the meaning of the fields, see the corresponding constants in +// distsqlrun/server.go. +message DistSQLVersionGossipInfo { + optional uint32 version = 1 [(gogoproto.nullable) = false, + (gogoproto.casttype) = "DistSQLVersion"]; + + optional uint32 min_accepted_version = 2 [(gogoproto.nullable) = false, + (gogoproto.casttype) = "DistSQLVersion"]; +} diff --git a/pkg/sql/distsqlrun/server.go b/pkg/sql/distsqlrun/server.go index cf45d9de913f..c028394132fd 100644 --- a/pkg/sql/distsqlrun/server.go +++ b/pkg/sql/distsqlrun/server.go @@ -23,6 +23,7 @@ import ( "golang.org/x/net/context" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" @@ -39,6 +40,9 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/uuid" ) +// DistSQLVersion identifies DistSQL engine versions. +type DistSQLVersion int + // Version identifies the distsqlrun protocol version. // // This version is separate from the main CockroachDB version numbering; it is @@ -62,11 +66,11 @@ import ( // - at some later point, we can choose to deprecate version 1 and have // servers only accept versions >= 2 (by setting // MinAcceptedVersion to 2). -const Version = 5 +const Version DistSQLVersion = 5 // MinAcceptedVersion is the oldest version that the server is // compatible with; see above. -const MinAcceptedVersion = 4 +const MinAcceptedVersion DistSQLVersion = 4 // workMemBytes specifies the maximum amount of memory in bytes a processor can // use. This limit is only observed if the use of temporary storage is enabled @@ -111,6 +115,9 @@ type ServerConfig struct { // JobRegistry manages jobs being used by this Server. JobRegistry *jobs.Registry + + // A handle to gossip used to broadcast the node's DistSQL version. + Gossip *gossip.Gossip } // ServerImpl implements the server for the distributed SQL APIs. @@ -172,9 +179,28 @@ func NewServer(ctx context.Context, cfg ServerConfig) *ServerImpl { // Start launches workers for the server. func (ds *ServerImpl) Start() { + // Gossip the version info so that other nodes don't plan incompatible flows + // for us. + if err := ds.ServerConfig.Gossip.AddInfoProto( + gossip.MakeDistSQLNodeVersionKey(ds.ServerConfig.NodeID.Get()), + &DistSQLVersionGossipInfo{ + Version: Version, + MinAcceptedVersion: MinAcceptedVersion, + }, + 0, // ttl - no expiration + ); err != nil { + panic(err) + } + ds.flowScheduler.Start() } +// FlowVerIsCompatible checks a flow's version is compatible with this node's +// DistSQL version. +func FlowVerIsCompatible(flowVer, minAcceptedVersion, serverVersion DistSQLVersion) bool { + return flowVer >= minAcceptedVersion && flowVer <= serverVersion +} + // Note: unless an error is returned, the returned context contains a span that // must be finished through Flow.Cleanup. func (ds *ServerImpl) setupFlow( @@ -183,8 +209,7 @@ func (ds *ServerImpl) setupFlow( req *SetupFlowRequest, syncFlowConsumer RowReceiver, ) (context.Context, *Flow, error) { - if req.Version < MinAcceptedVersion || - req.Version > Version { + if !FlowVerIsCompatible(req.Version, MinAcceptedVersion, Version) { err := errors.Errorf( "version mismatch in flow request: %d; this node accepts %d through %d", req.Version, MinAcceptedVersion, Version, diff --git a/pkg/sql/distsqlrun/server_test.go b/pkg/sql/distsqlrun/server_test.go index 50bd27fe8dd3..6f55819c95b8 100644 --- a/pkg/sql/distsqlrun/server_test.go +++ b/pkg/sql/distsqlrun/server_test.go @@ -22,6 +22,7 @@ import ( "golang.org/x/net/context" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/testutils" @@ -112,7 +113,7 @@ func TestServer(t *testing.T) { // Verify version handling. t.Run("version", func(t *testing.T) { testCases := []struct { - version uint32 + version DistSQLVersion expectedErr string }{ { @@ -147,3 +148,23 @@ func TestServer(t *testing.T) { } }) } + +// Test that a node gossips its DistSQL version information. +func TestDistSQLServerGossipsVersion(t *testing.T) { + defer leaktest.AfterTest(t)() + + s, _, _ := serverutils.StartServer(t, base.TestServerArgs{}) + defer s.Stopper().Stop(context.TODO()) + + var v DistSQLVersionGossipInfo + if err := s.Gossip().GetInfoProto( + gossip.MakeDistSQLNodeVersionKey(s.NodeID()), &v, + ); err != nil { + t.Fatal(err) + } + + if v.Version != Version || v.MinAcceptedVersion != MinAcceptedVersion { + t.Fatalf("node is gossipping the wrong version. Expected: [%d-%d], got [%d-%d", + Version, MinAcceptedVersion, v.Version, v.MinAcceptedVersion) + } +} diff --git a/pkg/sql/executor.go b/pkg/sql/executor.go index 685b6be2912f..f08179221b8f 100644 --- a/pkg/sql/executor.go +++ b/pkg/sql/executor.go @@ -330,6 +330,13 @@ type DistSQLPlannerTestingKnobs struct { // If OverrideSQLHealthCheck is set, we use this callback to get the health of // a node. OverrideHealthCheck func(node roachpb.NodeID, addrString string) error + // If OverrideDistSQLVersionCheck is set, the distSQLPlanner uses this instead + // of gossip for figuring out a node's DistSQL version. The callback is + // supposed to populate res, or it can return an error to simulate gossip not + // having any info for the node. If the test wants to simulate the node + // accepting any version, the callback can return a 0..+inf acceptable version + // range. + OverrideDistSQLVersionCheck func(node roachpb.NodeID, res *distsqlrun.DistSQLVersionGossipInfo) error } // NewExecutor creates an Executor and registers a callback on the @@ -372,6 +379,7 @@ func (e *Executor) Start( ctx = e.AnnotateCtx(ctx) log.Infof(ctx, "creating distSQLPlanner with address %s", nodeDesc.Address) e.distSQLPlanner = newDistSQLPlanner( + distsqlrun.Version, e.cfg.Settings, nodeDesc, e.cfg.RPCContext, diff --git a/pkg/sql/schema_changer.go b/pkg/sql/schema_changer.go index 651f245d30ea..ee288cca1994 100644 --- a/pkg/sql/schema_changer.go +++ b/pkg/sql/schema_changer.go @@ -925,6 +925,7 @@ func NewSchemaChangeManager( schemaChangers: make(map[sqlbase.ID]SchemaChanger), // TODO(radu): investigate using the same distSQLPlanner from the executor. distSQLPlanner: newDistSQLPlanner( + distsqlrun.Version, st, nodeDesc, rpcContext,