From 6e1bc57469ab8cbbd009d6cd56fbe41ce3937a70 Mon Sep 17 00:00:00 2001 From: Dan Upton Date: Tue, 9 May 2023 15:25:55 +0100 Subject: [PATCH 01/13] Controller Runtime --- agent/consul/server.go | 19 +- internal/controller/api.go | 188 +++++++++++++++ internal/controller/api_test.go | 268 ++++++++++++++++++++++ internal/controller/controller.go | 167 +++++++++++++- internal/controller/doc.go | 10 + internal/controller/lease.go | 5 + internal/controller/manager.go | 30 ++- internal/resource/demo/controller.go | 176 +++++++++++++- internal/resource/demo/controller_test.go | 102 ++++++++ internal/resource/demo/demo.go | 4 + 10 files changed, 952 insertions(+), 17 deletions(-) create mode 100644 internal/controller/api_test.go create mode 100644 internal/controller/doc.go create mode 100644 internal/resource/demo/controller_test.go diff --git a/agent/consul/server.go b/agent/consul/server.go index 3ec392daae69..73df01e858e3 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -507,7 +507,6 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom incomingRPCLimiter: incomingRPCLimiter, routineManager: routine.NewManager(logger.Named(logging.ConsulServer)), typeRegistry: resource.NewRegistry(), - controllerManager: controller.NewManager(logger.Named(logging.ControllerRuntime)), } incomingRPCLimiter.Register(s) @@ -783,6 +782,17 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom // to enable RPC forwarding. s.grpcHandler = newGRPCHandlerFromConfig(flat, config, s) s.grpcLeaderForwarder = flat.LeaderForwarder + + if err := s.setupInternalResourceService(logger); err != nil { + return nil, err + } + s.controllerManager = controller.NewManager( + s.internalResourceServiceClient, + logger.Named(logging.ControllerRuntime), + ) + s.registerResources() + go s.controllerManager.Run(&lib.StopChannelContext{StopCh: shutdownCh}) + go s.trackLeaderChanges() s.xdsCapacityController = xdscapacity.NewController(xdscapacity.Config{ @@ -792,10 +802,6 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom }) go s.xdsCapacityController.Run(&lib.StopChannelContext{StopCh: s.shutdownCh}) - if err := s.setupInternalResourceService(logger); err != nil { - return nil, err - } - // Initialize Autopilot. This must happen before starting leadership monitoring // as establishing leadership could attempt to use autopilot and cause a panic. s.initAutopilot(config) @@ -832,9 +838,6 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom return nil, err } - s.registerResources() - go s.controllerManager.Run(&lib.StopChannelContext{StopCh: shutdownCh}) - return s, nil } diff --git a/internal/controller/api.go b/internal/controller/api.go index 8545d339a7d6..d258eb40d6d8 100644 --- a/internal/controller/api.go +++ b/internal/controller/api.go @@ -1,6 +1,17 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + package controller import ( + "context" + "fmt" + "strings" + "time" + + "github.com/hashicorp/go-hclog" + + "github.com/hashicorp/consul/internal/resource" "github.com/hashicorp/consul/proto-public/pbresource" ) @@ -9,6 +20,86 @@ func ForType(managedType *pbresource.Type) Controller { return Controller{managedType: managedType} } +// WithReconciler changes the controller's reconciler. +func (c Controller) WithReconciler(reconciler Reconciler) Controller { + if reconciler == nil { + panic("reconciler must not be nil") + } + + c.reconciler = reconciler + return c +} + +// WithWatch adds a watch on the given type/dependency to the controller. mapper +// will be called to determine which resources must be reconciled as a result of +// a watched resource changing. +func (c Controller) WithWatch(watchedType *pbresource.Type, mapper DependencyMapper) Controller { + if watchedType == nil { + panic("watchedType must not be nil") + } + + if mapper == nil { + panic("mapper must not be nil") + } + + c.watches = append(c.watches, watch{watchedType, mapper}) + return c +} + +// WithLogger changes the controller's logger. +func (c Controller) WithLogger(logger hclog.Logger) Controller { + if logger == nil { + panic("logger must not be nil") + } + + c.logger = logger + return c +} + +// WithBackoff changes the base and maximum backoff values for the controller's +// retry rate limiter. +func (c Controller) WithBackoff(base, max time.Duration) Controller { + c.baseBackoff = base + c.maxBackoff = max + return c +} + +// WithPlacement changes where and how many replicas of the controller will run. +// In the majority of cases, the default placement (one leader elected instance +// per cluster) is the most appropriate and you shouldn't need to override it. +func (c Controller) WithPlacement(placement Placement) Controller { + c.placement = placement + return c +} + +// String returns a textual description of the controller, useful for debugging. +func (c Controller) String() string { + watchedTypes := make([]string, len(c.watches)) + for idx, w := range c.watches { + watchedTypes[idx] = fmt.Sprintf("%q", resource.ToGVK(w.watchedType)) + } + base, max := c.backoff() + return fmt.Sprintf( + ", placement=%q>", + resource.ToGVK(c.managedType), + strings.Join(watchedTypes, ", "), + base, max, + c.placement, + ) +} + +func (c Controller) backoff() (time.Duration, time.Duration) { + base := c.baseBackoff + if base == 0 { + base = 5 * time.Millisecond + } + max := c.maxBackoff + if max == 0 { + max = 1000 * time.Second + } + return base, max +} + // Controller runs a reconciliation loop to respond to changes in resources and // their dependencies. It is heavily inspired by Kubernetes' controller pattern: // https://kubernetes.io/docs/concepts/architecture/controller/ @@ -17,4 +108,101 @@ func ForType(managedType *pbresource.Type) Controller { // a controller, and then pass it to a Manager to be executed. type Controller struct { managedType *pbresource.Type + reconciler Reconciler + logger hclog.Logger + watches []watch + baseBackoff time.Duration + maxBackoff time.Duration + placement Placement +} + +type watch struct { + watchedType *pbresource.Type + mapper DependencyMapper +} + +// Request represents a request to reconcile the resource with the given ID. +type Request struct { + // ID of the resource that needs to be reconciled. + ID *pbresource.ID +} + +// Runtime contains the dependencies required by reconcilers. +type Runtime struct { + Client pbresource.ResourceServiceClient + Logger hclog.Logger +} + +// Reconciler implements the business logic of a controller. +type Reconciler interface { + // Reconcile the resource identified by req.ID. + Reconcile(ctx context.Context, rt Runtime, req Request) error +} + +// DependencyMapper is called when a dependency watched via WithWatch is changed +// to determine which of the controller's managed resources need to be reconciled. +type DependencyMapper func( + ctx context.Context, + rt Runtime, + res *pbresource.Resource, +) ([]Request, error) + +// MapOwner implements a DependencyMapper that returns the updated resource's owner. +func MapOwner(_ context.Context, _ Runtime, res *pbresource.Resource) ([]Request, error) { + var reqs []Request + if res.Owner != nil { + reqs = append(reqs, Request{ID: res.Owner}) + } + return reqs, nil +} + +// Placement determines where and how many replicas of the controller will run. +type Placement int + +const ( + // PlacementSingleton ensures there is a single, leader-elected, instance of + // the controller running in the cluster at any time. It's the default and is + // suitable for most use-cases. + PlacementSingleton Placement = iota + + // PlacementEachServer ensures there is a replica of the controller running on + // each server in the cluster. It is useful for cases where the controller is + // responsible for applying some configuration resource to the server whenever + // it changes (e.g. rate-limit configuration). Generally, controllers in this + // placement mode should not modify resources. + PlacementEachServer +) + +// String satisfies the fmt.Stringer interface. +func (p Placement) String() string { + switch p { + case PlacementSingleton: + return "singleton" + case PlacementEachServer: + return "each-server" + } + panic(fmt.Sprintf("unknown placement %d", p)) +} + +// RequeueAfterError is an error that allows a Reconciler to override the +// exponential backoff behavior of the Controller, rather than applying +// the backoff algorithm, returning a RequeueAfterError will cause the +// Controller to reschedule the Request at a given time in the future. +type RequeueAfterError time.Duration + +// Error implements the error interface. +func (r RequeueAfterError) Error() string { + return fmt.Sprintf("requeue at %s", time.Duration(r)) +} + +// RequeueAfter constructs a RequeueAfterError with the given duration +// setting. +func RequeueAfter(after time.Duration) error { + return RequeueAfterError(after) +} + +// RequeueNow constructs a RequeueAfterError that reschedules the Request +// immediately. +func RequeueNow() error { + return RequeueAfterError(0) } diff --git a/internal/controller/api_test.go b/internal/controller/api_test.go new file mode 100644 index 000000000000..2006664b20fb --- /dev/null +++ b/internal/controller/api_test.go @@ -0,0 +1,268 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package controller_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + + svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing" + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/resource/demo" + "github.com/hashicorp/consul/proto-public/pbresource" + "github.com/hashicorp/consul/proto/private/prototest" + "github.com/hashicorp/consul/sdk/testutil" +) + +func TestController_API(t *testing.T) { + t.Parallel() + + rec := newTestReconciler() + client := svctest.RunResourceService(t, demo.RegisterTypes) + + ctrl := controller. + ForType(demo.TypeV2Artist). + WithWatch(demo.TypeV2Album, controller.MapOwner). + WithBackoff(10*time.Millisecond, 100*time.Millisecond). + WithReconciler(rec) + + mgr := controller.NewManager(client, testutil.Logger(t)) + mgr.Register(ctrl) + mgr.SetRaftLeader(true) + go mgr.Run(testContext(t)) + + t.Run("managed resource type", func(t *testing.T) { + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + + rsp, err := client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + req := rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + }) + + t.Run("watched resource type", func(t *testing.T) { + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + + rsp, err := client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + req := rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + + rec.expectNoRequest(t, 500*time.Millisecond) + + album, err := demo.GenerateV2Album(rsp.Resource.Id) + require.NoError(t, err) + + _, err = client.Write(testContext(t), &pbresource.WriteRequest{Resource: album}) + require.NoError(t, err) + + req = rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + }) + + t.Run("error retries", func(t *testing.T) { + rec.failNext(errors.New("KABOOM")) + + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + + rsp, err := client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + req := rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + + // Reconciler should be called with the same request again. + req = rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + }) + + t.Run("panic retries", func(t *testing.T) { + rec.panicNext("KABOOM") + + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + + rsp, err := client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + req := rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + + // Reconciler should be called with the same request again. + req = rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + }) + + t.Run("defer", func(t *testing.T) { + rec.failNext(controller.RequeueAfter(1 * time.Second)) + + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + + rsp, err := client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + req := rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + + rec.expectNoRequest(t, 750*time.Millisecond) + + req = rec.wait(t) + prototest.AssertDeepEqual(t, rsp.Resource.Id, req.ID) + }) +} + +func TestController_Placement(t *testing.T) { + t.Parallel() + + t.Run("singleton", func(t *testing.T) { + rec := newTestReconciler() + client := svctest.RunResourceService(t, demo.RegisterTypes) + + ctrl := controller. + ForType(demo.TypeV2Artist). + WithWatch(demo.TypeV2Album, controller.MapOwner). + WithPlacement(controller.PlacementSingleton). + WithReconciler(rec) + + mgr := controller.NewManager(client, testutil.Logger(t)) + mgr.Register(ctrl) + go mgr.Run(testContext(t)) + + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + + // Reconciler should not be called until we're the Raft leader. + _, err = client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + rec.expectNoRequest(t, 500*time.Millisecond) + + // Become the leader and check the reconciler is called. + mgr.SetRaftLeader(true) + _ = rec.wait(t) + + // Should not be called after losing leadership. + mgr.SetRaftLeader(false) + _, err = client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + rec.expectNoRequest(t, 500*time.Millisecond) + }) + + t.Run("each server", func(t *testing.T) { + rec := newTestReconciler() + client := svctest.RunResourceService(t, demo.RegisterTypes) + + ctrl := controller. + ForType(demo.TypeV2Artist). + WithWatch(demo.TypeV2Album, controller.MapOwner). + WithPlacement(controller.PlacementEachServer). + WithReconciler(rec) + + mgr := controller.NewManager(client, testutil.Logger(t)) + mgr.Register(ctrl) + go mgr.Run(testContext(t)) + + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + + // Reconciler should be called even though we're not the Raft leader. + _, err = client.Write(testContext(t), &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + _ = rec.wait(t) + }) +} + +func TestController_String(t *testing.T) { + ctrl := controller. + ForType(demo.TypeV2Artist). + WithWatch(demo.TypeV2Album, controller.MapOwner). + WithBackoff(5*time.Second, 1*time.Hour). + WithPlacement(controller.PlacementEachServer) + + require.Equal(t, + `, placement="each-server">`, + ctrl.String(), + ) +} + +func TestController_NoReconciler(t *testing.T) { + client := svctest.RunResourceService(t, demo.RegisterTypes) + mgr := controller.NewManager(client, testutil.Logger(t)) + + ctrl := controller.ForType(demo.TypeV2Artist) + require.PanicsWithValue(t, + `cannot register controller without a reconciler , placement="singleton">`, + func() { mgr.Register(ctrl) }) +} + +func newTestReconciler() *testReconciler { + return &testReconciler{ + calls: make(chan controller.Request), + errors: make(chan error, 1), + panics: make(chan any, 1), + } +} + +type testReconciler struct { + calls chan controller.Request + errors chan error + panics chan any +} + +func (r *testReconciler) Reconcile(_ context.Context, _ controller.Runtime, req controller.Request) error { + r.calls <- req + + select { + case err := <-r.errors: + return err + case p := <-r.panics: + panic(p) + default: + return nil + } +} + +func (r *testReconciler) failNext(err error) { r.errors <- err } +func (r *testReconciler) panicNext(p any) { r.panics <- p } + +func (r *testReconciler) expectNoRequest(t *testing.T, duration time.Duration) { + t.Helper() + + started := time.Now() + select { + case req := <-r.calls: + t.Fatalf("expected no request for %s, but got: %s after %s", duration, req.ID, time.Since(started)) + case <-time.After(duration): + } +} + +func (r *testReconciler) wait(t *testing.T) controller.Request { + t.Helper() + + var req controller.Request + select { + case req = <-r.calls: + case <-time.After(500 * time.Millisecond): + t.Fatal("Reconcile was not called after 500ms") + } + return req +} + +func testContext(t *testing.T) context.Context { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + return ctx +} diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 11933b39facc..d99ca26f0d3f 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -1,15 +1,29 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + package controller import ( "context" + "errors" + "fmt" + "time" "github.com/hashicorp/go-hclog" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" + + "github.com/hashicorp/consul/agent/consul/controller/queue" + "github.com/hashicorp/consul/internal/resource" + "github.com/hashicorp/consul/internal/storage" + "github.com/hashicorp/consul/proto-public/pbresource" ) // controllerRunner contains the actual implementation of running a controller // including creating watches, calling the reconciler, handling retries, etc. type controllerRunner struct { ctrl Controller + client pbresource.ResourceServiceClient logger hclog.Logger } @@ -17,6 +31,155 @@ func (c *controllerRunner) run(ctx context.Context) error { c.logger.Debug("controller running") defer c.logger.Debug("controller stopping") - <-ctx.Done() - return ctx.Err() + group, groupCtx := errgroup.WithContext(ctx) + recQueue := runQueue[Request](groupCtx, c.ctrl) + + // Managed Type Events → Reconciliation Queue + group.Go(func() error { + return c.watch(groupCtx, c.ctrl.managedType, func(res *pbresource.Resource) { + recQueue.Add(Request{ID: res.Id}) + }) + }) + + for _, watch := range c.ctrl.watches { + watch := watch + mapQueue := runQueue[*pbresource.Resource](groupCtx, c.ctrl) + + // Watched Type Events → Mapper Queue + group.Go(func() error { + return c.watch(groupCtx, watch.watchedType, mapQueue.Add) + }) + + // Mapper Queue → Mapper → Reconciliation Queue + group.Go(func() error { + return c.runMapper(groupCtx, watch, mapQueue, recQueue) + }) + } + + // Reconciliation Queue → Reconciler + group.Go(func() error { + return c.runReconciler(groupCtx, recQueue) + }) + + return group.Wait() +} + +func runQueue[T queue.ItemType](ctx context.Context, ctrl Controller) queue.WorkQueue[T] { + base, max := ctrl.backoff() + return queue.RunWorkQueue[T](ctx, base, max) +} + +func (c *controllerRunner) watch(ctx context.Context, typ *pbresource.Type, add func(*pbresource.Resource)) error { + watch, err := c.client.WatchList(ctx, &pbresource.WatchListRequest{ + Type: typ, + Tenancy: &pbresource.Tenancy{ + Partition: storage.Wildcard, + PeerName: storage.Wildcard, + Namespace: storage.Wildcard, + }, + }) + if err != nil { + c.logger.Error("failed to create watch", "error", err) + return err + } + + for { + event, err := watch.Recv() + if err != nil { + c.logger.Warn("error received from watch", "error", err) + return err + } + add(event.Resource) + } +} + +func (c *controllerRunner) runMapper( + ctx context.Context, + w watch, + from queue.WorkQueue[*pbresource.Resource], + to queue.WorkQueue[Request], +) error { + logger := c.logger.With("watched_resource_type", resource.ToGVK(w.watchedType)) + + for { + res, shutdown := from.Get() + if shutdown { + return nil + } + + var reqs []Request + err := c.handlePanic(func() error { + var err error + reqs, err = w.mapper(ctx, c.runtime(), res) + return err + }) + if err != nil { + from.AddRateLimited(res) + from.Done(res) + continue + } + + for _, r := range reqs { + if !proto.Equal(r.ID.Type, c.ctrl.managedType) { + logger.Error("dependency mapper returned request for a resource of the wrong type", + "type_expected", resource.ToGVK(c.ctrl.managedType), + "type_got", resource.ToGVK(r.ID.Type), + ) + continue + } + to.Add(r) + } + + from.Forget(res) + from.Done(res) + } +} + +func (c *controllerRunner) runReconciler(ctx context.Context, queue queue.WorkQueue[Request]) error { + for { + req, shutdown := queue.Get() + if shutdown { + return nil + } + + c.logger.Trace("handling request", "request", req) + err := c.handlePanic(func() error { + return c.ctrl.reconciler.Reconcile(ctx, c.runtime(), req) + }) + if err == nil { + queue.Forget(req) + } else { + var requeueAfter RequeueAfterError + if errors.As(err, &requeueAfter) { + queue.Forget(req) + queue.AddAfter(req, time.Duration(requeueAfter)) + } else { + queue.AddRateLimited(req) + } + } + queue.Done(req) + } +} + +func (c *controllerRunner) handlePanic(fn func() error) (err error) { + defer func() { + if r := recover(); r != nil { + stack := hclog.Stacktrace() + c.logger.Error("controller panic", + "panic", r, + "stack", stack, + ) + err = fmt.Errorf("panic [recovered]: %v", r) + return + } + }() + + return fn() +} + +func (c *controllerRunner) runtime() Runtime { + return Runtime{ + Client: c.client, + Logger: c.logger, + } } diff --git a/internal/controller/doc.go b/internal/controller/doc.go new file mode 100644 index 000000000000..28953791525c --- /dev/null +++ b/internal/controller/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// Package controller provides an API for implementing control loops on top of +// Consul resources. It is heavily inspired by [Kubebuilder] and the Kubernetes +// [controller runtime]. +// +// [Kubebuilder]: https://github.com/kubernetes-sigs/kubebuilder +// [controller runtime]: https://github.com/kubernetes-sigs/controller-runtime +package controller diff --git a/internal/controller/lease.go b/internal/controller/lease.go index 33e284a69c7c..2cb00d133019 100644 --- a/internal/controller/lease.go +++ b/internal/controller/lease.go @@ -22,3 +22,8 @@ type raftLease struct { func (l *raftLease) Held() bool { return l.m.raftLeader.Load() } func (l *raftLease) Changed() <-chan struct{} { return l.ch } + +type eternalLease struct{} + +func (eternalLease) Held() bool { return true } +func (eternalLease) Changed() <-chan struct{} { return nil } diff --git a/internal/controller/manager.go b/internal/controller/manager.go index 90b9f2994bae..92c5829c581e 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -2,16 +2,19 @@ package controller import ( "context" + "fmt" "sync" "sync/atomic" "github.com/hashicorp/go-hclog" "github.com/hashicorp/consul/internal/resource" + "github.com/hashicorp/consul/proto-public/pbresource" ) // Manager is responsible for scheduling the execution of controllers. type Manager struct { + client pbresource.ResourceServiceClient logger hclog.Logger raftLeader atomic.Bool @@ -24,8 +27,11 @@ type Manager struct { // NewManager creates a Manager. logger will be used by the Manager, and as the // base logger for controllers when one is not specified using WithLogger. -func NewManager(logger hclog.Logger) *Manager { - return &Manager{logger: logger} +func NewManager(client pbresource.ResourceServiceClient, logger hclog.Logger) *Manager { + return &Manager{ + client: client, + logger: logger, + } } // Register the given controller to be executed by the Manager. Cannot be called @@ -38,6 +44,10 @@ func (m *Manager) Register(ctrl Controller) { panic("cannot register additional controllers after calling Run") } + if ctrl.reconciler == nil { + panic(fmt.Sprintf("cannot register controller without a reconciler %s", ctrl)) + } + m.controllers = append(m.controllers, ctrl) } @@ -53,11 +63,17 @@ func (m *Manager) Run(ctx context.Context) { m.running = true for _, desc := range m.controllers { + logger := desc.logger + if logger == nil { + logger = m.logger.With("managed_type", resource.ToGVK(desc.managedType)) + } + runner := &controllerRunner{ ctrl: desc, - logger: m.logger.With("managed_type", resource.ToGVK(desc.managedType)), + client: m.client, + logger: logger, } - go newSupervisor(runner.run, m.newLeaseLocked()).run(ctx) + go newSupervisor(runner.run, m.newLeaseLocked(desc)).run(ctx) } } @@ -82,7 +98,11 @@ func (m *Manager) SetRaftLeader(leader bool) { } } -func (m *Manager) newLeaseLocked() Lease { +func (m *Manager) newLeaseLocked(ctrl Controller) Lease { + if ctrl.placement == PlacementEachServer { + return eternalLease{} + } + ch := make(chan struct{}, 1) m.leaseChans = append(m.leaseChans, ch) return &raftLease{m: m, ch: ch} diff --git a/internal/resource/demo/controller.go b/internal/resource/demo/controller.go index f2172f0f852c..11de1c5057ee 100644 --- a/internal/resource/demo/controller.go +++ b/internal/resource/demo/controller.go @@ -1,6 +1,25 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + package demo -import "github.com/hashicorp/consul/internal/controller" +import ( + "context" + "fmt" + "math/rand" + + "github.com/oklog/ulid/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/resource" + "github.com/hashicorp/consul/proto-public/pbresource" + pbdemov2 "github.com/hashicorp/consul/proto/private/pbdemo/v2" +) + +const statusKeyArtistController = "consul.io/artist-controller" // RegisterControllers registers controllers for the demo types. Should only be // called in dev mode. @@ -9,5 +28,158 @@ func RegisterControllers(mgr *controller.Manager) { } func artistController() controller.Controller { - return controller.ForType(TypeV2Artist) + return controller.ForType(TypeV2Artist). + WithWatch(TypeV2Album, controller.MapOwner). + WithReconciler(&artistReconciler{}) +} + +type artistReconciler struct{} + +func (r *artistReconciler) Reconcile(ctx context.Context, rt controller.Runtime, req controller.Request) error { + rsp, err := rt.Client.Read(ctx, &pbresource.ReadRequest{Id: req.ID}) + switch { + case status.Code(err) == codes.NotFound: + return nil + case err != nil: + return err + } + res := rsp.Resource + + var artist pbdemov2.Artist + if err := res.Data.UnmarshalTo(&artist); err != nil { + return err + } + conditions := []*pbresource.Condition{ + { + Type: "Accepted", + State: pbresource.Condition_STATE_TRUE, + Reason: "Accepted", + Message: fmt.Sprintf("Artist '%s' accepted", artist.Name), + }, + } + + numAlbums := 3 + if artist.Genre == pbdemov2.Genre_GENRE_BLUES { + numAlbums = 10 + } + + desiredAlbums, err := generateV2AlbumsDeterministic(res.Id, numAlbums) + if err != nil { + return err + } + + actualAlbums, err := rt.Client.List(ctx, &pbresource.ListRequest{ + Type: TypeV2Album, + Tenancy: res.Id.Tenancy, + NamePrefix: fmt.Sprintf("%s/", res.Id.Name), + }) + if err != nil { + return err + } + + writes, deletions, err := diffAlbums(desiredAlbums, actualAlbums.Resources) + if err != nil { + return err + } + for _, w := range writes { + if _, err := rt.Client.Write(ctx, &pbresource.WriteRequest{Resource: w}); err != nil { + return err + } + } + for _, d := range deletions { + if _, err := rt.Client.Delete(ctx, &pbresource.DeleteRequest{Id: d}); err != nil { + return err + } + } + + for _, want := range desiredAlbums { + var album pbdemov2.Album + if err := want.Data.UnmarshalTo(&album); err != nil { + return err + } + conditions = append(conditions, &pbresource.Condition{ + Type: "AlbumCreated", + State: pbresource.Condition_STATE_TRUE, + Reason: "AlbumCreated", + Message: fmt.Sprintf("Album '%s' created for artist '%s'", album.Title, artist.Name), + Resource: resource.Reference(want.Id, ""), + }) + } + + newStatus := &pbresource.Status{ + ObservedGeneration: res.Generation, + Conditions: conditions, + } + + if proto.Equal(res.Status[statusKeyArtistController], newStatus) { + return nil + } + + _, err = rt.Client.WriteStatus(ctx, &pbresource.WriteStatusRequest{ + Id: res.Id, + Key: statusKeyArtistController, + Status: newStatus, + }) + return err +} + +func diffAlbums(want, have []*pbresource.Resource) ([]*pbresource.Resource, []*pbresource.ID, error) { + haveMap := make(map[string]*pbresource.Resource, len(have)) + for _, r := range have { + haveMap[r.Id.Name] = r + } + + wantMap := make(map[string]struct{}, len(want)) + for _, r := range want { + wantMap[r.Id.Name] = struct{}{} + } + + writes := make([]*pbresource.Resource, 0) + for _, w := range want { + h, ok := haveMap[w.Id.Name] + if ok { + var wd, hd pbdemov2.Album + if err := w.Data.UnmarshalTo(&wd); err != nil { + return nil, nil, err + } + if err := h.Data.UnmarshalTo(&hd); err != nil { + return nil, nil, err + } + if proto.Equal(&wd, &hd) { + continue + } + } + + writes = append(writes, w) + } + + deletions := make([]*pbresource.ID, 0) + for _, h := range have { + if _, ok := wantMap[h.Id.Name]; ok { + continue + } + deletions = append(deletions, h.Id) + } + + return writes, deletions, nil +} + +func generateV2AlbumsDeterministic(artistID *pbresource.ID, count int) ([]*pbresource.Resource, error) { + uid, err := ulid.Parse(artistID.Uid) + if err != nil { + return nil, fmt.Errorf("failed to parse Uid: %w", err) + } + rand := rand.New(rand.NewSource(int64(uid.Time()))) + + albums := make([]*pbresource.Resource, count) + for i := 0; i < count; i++ { + album, err := generateV2Album(artistID, rand) + if err != nil { + return nil, err + } + // Add suffix to avoid collisions. + album.Id.Name = fmt.Sprintf("%s-%d", album.Id.Name, i) + albums[i] = album + } + return albums, nil } diff --git a/internal/resource/demo/controller_test.go b/internal/resource/demo/controller_test.go new file mode 100644 index 000000000000..8d4ee79c73e0 --- /dev/null +++ b/internal/resource/demo/controller_test.go @@ -0,0 +1,102 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package demo + +import ( + "testing" + + "github.com/stretchr/testify/require" + + svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing" + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/proto-public/pbresource" + pbdemov2 "github.com/hashicorp/consul/proto/private/pbdemo/v2" + "github.com/hashicorp/consul/sdk/testutil" +) + +func TestArtistReconciler(t *testing.T) { + client := svctest.RunResourceService(t, RegisterTypes) + + // Seed the database with an artist. + res, err := GenerateV2Artist() + require.NoError(t, err) + + // Set the genre to BLUES to ensure there are 10 albums. + var artist pbdemov2.Artist + require.NoError(t, res.Data.UnmarshalTo(&artist)) + artist.Genre = pbdemov2.Genre_GENRE_BLUES + require.NoError(t, res.Data.MarshalFrom(&artist)) + + ctx := testutil.TestContext(t) + writeRsp, err := client.Write(ctx, &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + // Call the reconciler for that artist. + var rec artistReconciler + runtime := controller.Runtime{ + Client: client, + Logger: testutil.Logger(t), + } + req := controller.Request{ + ID: writeRsp.Resource.Id, + } + require.NoError(t, rec.Reconcile(ctx, runtime, req)) + + // Check the status was updated. + readRsp, err := client.Read(ctx, &pbresource.ReadRequest{Id: writeRsp.Resource.Id}) + require.NoError(t, err) + require.Contains(t, readRsp.Resource.Status, "consul.io/artist-controller") + + status := readRsp.Resource.Status["consul.io/artist-controller"] + require.Equal(t, writeRsp.Resource.Generation, status.ObservedGeneration) + require.Len(t, status.Conditions, 11) + require.Equal(t, "Accepted", status.Conditions[0].Type) + require.Equal(t, "AlbumCreated", status.Conditions[1].Type) + + // Check the albums were created. + listRsp, err := client.List(ctx, &pbresource.ListRequest{ + Type: TypeV2Album, + Tenancy: readRsp.Resource.Id.Tenancy, + }) + require.NoError(t, err) + require.Len(t, listRsp.Resources, 10) + + // Delete an album. + _, err = client.Delete(ctx, &pbresource.DeleteRequest{Id: listRsp.Resources[0].Id}) + require.NoError(t, err) + + // Call the reconciler again. + require.NoError(t, rec.Reconcile(ctx, runtime, req)) + + // Check the album was recreated. + listRsp, err = client.List(ctx, &pbresource.ListRequest{ + Type: TypeV2Album, + Tenancy: readRsp.Resource.Id.Tenancy, + }) + require.NoError(t, err) + require.Len(t, listRsp.Resources, 10) + + // Set the genre to DISCO. + readRsp, err = client.Read(ctx, &pbresource.ReadRequest{Id: writeRsp.Resource.Id}) + require.NoError(t, err) + + res = readRsp.Resource + require.NoError(t, res.Data.UnmarshalTo(&artist)) + artist.Genre = pbdemov2.Genre_GENRE_DISCO + require.NoError(t, res.Data.MarshalFrom(&artist)) + + _, err = client.Write(ctx, &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + // Call the reconciler again. + require.NoError(t, rec.Reconcile(ctx, runtime, req)) + + // Check there are only 3 albums now. + listRsp, err = client.List(ctx, &pbresource.ListRequest{ + Type: TypeV2Album, + Tenancy: readRsp.Resource.Id.Tenancy, + }) + require.NoError(t, err) + require.Len(t, listRsp.Resources, 3) +} diff --git a/internal/resource/demo/demo.go b/internal/resource/demo/demo.go index fde9272aeb02..842b75739bc4 100644 --- a/internal/resource/demo/demo.go +++ b/internal/resource/demo/demo.go @@ -204,6 +204,10 @@ func GenerateV2Artist() (*pbresource.Resource, error) { // GenerateV2Album generates a random Album resource, owned by the Artist with // the given ID. func GenerateV2Album(artistID *pbresource.ID) (*pbresource.Resource, error) { + return generateV2Album(artistID, rand.New(rand.NewSource(time.Now().UnixNano()))) +} + +func generateV2Album(artistID *pbresource.ID, rand *rand.Rand) (*pbresource.Resource, error) { adjective := adjectives[rand.Intn(len(adjectives))] noun := nouns[rand.Intn(len(nouns))] From 5f079eb05b3f43b59bfce182547ebcccda2e61f8 Mon Sep 17 00:00:00 2001 From: Dan Bond Date: Tue, 9 May 2023 09:44:31 -0700 Subject: [PATCH 02/13] Revert "ci: remove test splitting for compatibility tests (#17166)" (#17262) This reverts commit 861a8151d50377315c6c391833fef85b71b54d18. --- .github/workflows/test-integrations.yml | 45 +++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 3615e1554805..33d1ccc9e207 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -237,11 +237,47 @@ jobs: name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} path: ${{ env.TEST_RESULTS_DIR }} + generate-compatibility-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Compatibility Job Matrices + outputs: + compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Compatibility Job Matrix + id: set-matrix + env: + TOTAL_RUNNERS: 6 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + cd ./test/integration/consul-container + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(find ./test -mindepth 1 -maxdepth 2 -type d | wc -l) + + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$((NUM_DIRS-1)) + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) + { + echo -n "compatibility-matrix=" + find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \ + | grep -v util | grep -v upgrade \ + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join(" "))' + } >> "$GITHUB_OUTPUT" compatibility-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} needs: - setup - dev-build + - generate-compatibility-job-matrices + strategy: + fail-fast: false + matrix: + test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 @@ -271,9 +307,12 @@ jobs: mkdir -p "/tmp/test-results" cd ./test/integration/consul-container docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's, ,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ - --format=standard-verbose \ + --format=short-verbose \ --debug \ --rerun-fails=3 \ -- \ @@ -282,7 +321,7 @@ jobs: -tags "${{ env.GOTAGS }}" \ -timeout=30m \ -json \ - `go list ./... | grep -v upgrade` \ + ${{ matrix.test-cases }} \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ @@ -383,6 +422,7 @@ jobs: --raw-command \ --format=short-verbose \ --debug \ + --rerun-fails=3 \ --packages="./..." \ -- \ go test \ @@ -416,6 +456,7 @@ jobs: - vault-integration-test - generate-envoy-job-matrices - envoy-integration-test + - generate-compatibility-job-matrices - compatibility-integration-test - generate-upgrade-job-matrices - upgrade-integration-test From 972998203ea7056d93de38a27525ba57eaae46c7 Mon Sep 17 00:00:00 2001 From: Dan Upton Date: Tue, 9 May 2023 18:14:20 +0100 Subject: [PATCH 03/13] controller: deduplicate items in queue (#17168) --- agent/consul/controller/queue/defer.go | 12 +++----- agent/consul/controller/queue/queue.go | 27 ++++++++-------- agent/consul/controller/queue/rate.go | 12 +++----- agent/consul/controller/queue/rate_test.go | 2 ++ agent/consul/controller/queue_test.go | 2 -- agent/consul/controller/reconciler.go | 12 ++++++++ internal/controller/api.go | 13 ++++++++ internal/controller/controller.go | 36 ++++++++++++++++------ 8 files changed, 79 insertions(+), 37 deletions(-) diff --git a/agent/consul/controller/queue/defer.go b/agent/consul/controller/queue/defer.go index d6e261288f58..01666219c291 100644 --- a/agent/consul/controller/queue/defer.go +++ b/agent/consul/controller/queue/defer.go @@ -41,7 +41,7 @@ type deferredRequest[T ItemType] struct { // future processing type deferQueue[T ItemType] struct { heap *deferHeap[T] - entries map[T]*deferredRequest[T] + entries map[string]*deferredRequest[T] addChannel chan *deferredRequest[T] heartbeat *time.Ticker @@ -55,7 +55,7 @@ func NewDeferQueue[T ItemType](tick time.Duration) DeferQueue[T] { return &deferQueue[T]{ heap: dHeap, - entries: make(map[T]*deferredRequest[T]), + entries: make(map[string]*deferredRequest[T]), addChannel: make(chan *deferredRequest[T]), heartbeat: time.NewTicker(tick), } @@ -78,7 +78,7 @@ func (q *deferQueue[T]) Defer(ctx context.Context, item T, until time.Time) { // deferEntry adds a deferred request to the priority queue func (q *deferQueue[T]) deferEntry(entry *deferredRequest[T]) { - existing, exists := q.entries[entry.item] + existing, exists := q.entries[entry.item.Key()] if exists { // insert or update the item deferral time if existing.enqueueAt.After(entry.enqueueAt) { @@ -90,7 +90,7 @@ func (q *deferQueue[T]) deferEntry(entry *deferredRequest[T]) { } heap.Push(q.heap, entry) - q.entries[entry.item] = entry + q.entries[entry.item.Key()] = entry } // readyRequest returns a pointer to the next ready Request or @@ -108,7 +108,7 @@ func (q *deferQueue[T]) readyRequest() *T { } entry = heap.Pop(q.heap).(*deferredRequest[T]) - delete(q.entries, entry.item) + delete(q.entries, entry.item.Key()) return &entry.item } @@ -182,8 +182,6 @@ func (q *deferQueue[T]) Process(ctx context.Context, callback func(item T)) { } } -var _ heap.Interface = &deferHeap[string]{} - // deferHeap implements heap.Interface type deferHeap[T ItemType] []*deferredRequest[T] diff --git a/agent/consul/controller/queue/queue.go b/agent/consul/controller/queue/queue.go index fd712b40a4c0..6d9f0a657125 100644 --- a/agent/consul/controller/queue/queue.go +++ b/agent/consul/controller/queue/queue.go @@ -13,7 +13,10 @@ import ( // https://github.com/kubernetes/client-go/blob/release-1.25/util/workqueue/queue.go // ItemType is the type constraint for items in the WorkQueue. -type ItemType comparable +type ItemType interface { + // Key returns a string that will be used to de-duplicate items in the queue. + Key() string +} // WorkQueue is an interface for a work queue with semantics to help with // retries and rate limiting. @@ -43,9 +46,9 @@ type queue[T ItemType] struct { // dirty holds the working set of all Requests, whether they are being // processed or not - dirty map[T]struct{} + dirty map[string]struct{} // processing holds the set of current requests being processed - processing map[T]struct{} + processing map[string]struct{} // deferred is an internal priority queue that tracks deferred // Requests @@ -66,8 +69,8 @@ type queue[T ItemType] struct { func RunWorkQueue[T ItemType](ctx context.Context, baseBackoff, maxBackoff time.Duration) WorkQueue[T] { q := &queue[T]{ ratelimiter: NewRateLimiter[T](baseBackoff, maxBackoff), - dirty: make(map[T]struct{}), - processing: make(map[T]struct{}), + dirty: make(map[string]struct{}), + processing: make(map[string]struct{}), cond: sync.NewCond(&sync.Mutex{}), deferred: NewDeferQueue[T](500 * time.Millisecond), ctx: ctx, @@ -115,8 +118,8 @@ func (q *queue[T]) Get() (item T, shutdown bool) { item, q.queue = q.queue[0], q.queue[1:] - q.processing[item] = struct{}{} - delete(q.dirty, item) + q.processing[item.Key()] = struct{}{} + delete(q.dirty, item.Key()) return item, false } @@ -129,12 +132,12 @@ func (q *queue[T]) Add(item T) { if q.shuttingDown() { return } - if _, ok := q.dirty[item]; ok { + if _, ok := q.dirty[item.Key()]; ok { return } - q.dirty[item] = struct{}{} - if _, ok := q.processing[item]; ok { + q.dirty[item.Key()] = struct{}{} + if _, ok := q.processing[item.Key()]; ok { return } @@ -175,8 +178,8 @@ func (q *queue[T]) Done(item T) { q.cond.L.Lock() defer q.cond.L.Unlock() - delete(q.processing, item) - if _, ok := q.dirty[item]; ok { + delete(q.processing, item.Key()) + if _, ok := q.dirty[item.Key()]; ok { q.queue = append(q.queue, item) q.cond.Signal() } diff --git a/agent/consul/controller/queue/rate.go b/agent/consul/controller/queue/rate.go index f4f0dc5ad9a0..471601f85a27 100644 --- a/agent/consul/controller/queue/rate.go +++ b/agent/consul/controller/queue/rate.go @@ -22,10 +22,8 @@ type Limiter[T ItemType] interface { Forget(request T) } -var _ Limiter[string] = &ratelimiter[string]{} - type ratelimiter[T ItemType] struct { - failures map[T]int + failures map[string]int base time.Duration max time.Duration mutex sync.RWMutex @@ -35,7 +33,7 @@ type ratelimiter[T ItemType] struct { // backoff. func NewRateLimiter[T ItemType](base, max time.Duration) Limiter[T] { return &ratelimiter[T]{ - failures: make(map[T]int), + failures: make(map[string]int), base: base, max: max, } @@ -47,8 +45,8 @@ func (r *ratelimiter[T]) NextRetry(request T) time.Duration { r.mutex.RLock() defer r.mutex.RUnlock() - exponent := r.failures[request] - r.failures[request] = r.failures[request] + 1 + exponent := r.failures[request.Key()] + r.failures[request.Key()] = r.failures[request.Key()] + 1 backoff := float64(r.base.Nanoseconds()) * math.Pow(2, float64(exponent)) // make sure we don't overflow time.Duration @@ -69,5 +67,5 @@ func (r *ratelimiter[T]) Forget(request T) { r.mutex.Lock() defer r.mutex.Unlock() - delete(r.failures, request) + delete(r.failures, request.Key()) } diff --git a/agent/consul/controller/queue/rate_test.go b/agent/consul/controller/queue/rate_test.go index f44df15e1b4d..40dc540138e2 100644 --- a/agent/consul/controller/queue/rate_test.go +++ b/agent/consul/controller/queue/rate_test.go @@ -12,6 +12,8 @@ import ( type Request struct{ Kind string } +func (r Request) Key() string { return r.Kind } + func TestRateLimiter_Backoff(t *testing.T) { t.Parallel() diff --git a/agent/consul/controller/queue_test.go b/agent/consul/controller/queue_test.go index 867dfeff134b..11e1bc82b762 100644 --- a/agent/consul/controller/queue_test.go +++ b/agent/consul/controller/queue_test.go @@ -10,8 +10,6 @@ import ( "github.com/hashicorp/consul/agent/consul/controller/queue" ) -var _ queue.WorkQueue[string] = &countingWorkQueue[string]{} - type countingWorkQueue[T queue.ItemType] struct { getCounter uint64 addCounter uint64 diff --git a/agent/consul/controller/reconciler.go b/agent/consul/controller/reconciler.go index ce0c6e97a3a9..dc4222508b57 100644 --- a/agent/consul/controller/reconciler.go +++ b/agent/consul/controller/reconciler.go @@ -20,6 +20,18 @@ type Request struct { Meta *acl.EnterpriseMeta } +// Key satisfies the queue.ItemType interface. It returns a string which will be +// used to de-duplicate requests in the queue. +func (r Request) Key() string { + return fmt.Sprintf( + `kind=%q,name=%q,part=%q,ns=%q`, + r.Kind, + r.Name, + r.Meta.PartitionOrDefault(), + r.Meta.NamespaceOrDefault(), + ) +} + // RequeueAfterError is an error that allows a Reconciler to override the // exponential backoff behavior of the Controller, rather than applying // the backoff algorithm, returning a RequeueAfterError will cause the diff --git a/internal/controller/api.go b/internal/controller/api.go index d258eb40d6d8..7a2e89be4641 100644 --- a/internal/controller/api.go +++ b/internal/controller/api.go @@ -127,6 +127,19 @@ type Request struct { ID *pbresource.ID } +// Key satisfies the queue.ItemType interface. It returns a string which will be +// used to de-duplicate requests in the queue. +func (r Request) Key() string { + return fmt.Sprintf( + "part=%q,peer=%q,ns=%q,name=%q,uid=%q", + r.ID.Tenancy.Partition, + r.ID.Tenancy.PeerName, + r.ID.Tenancy.Namespace, + r.ID.Name, + r.ID.Uid, + ) +} + // Runtime contains the dependencies required by reconcilers. type Runtime struct { Client pbresource.ResourceServiceClient diff --git a/internal/controller/controller.go b/internal/controller/controller.go index d99ca26f0d3f..296ff5faf4f5 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -43,11 +43,13 @@ func (c *controllerRunner) run(ctx context.Context) error { for _, watch := range c.ctrl.watches { watch := watch - mapQueue := runQueue[*pbresource.Resource](groupCtx, c.ctrl) + mapQueue := runQueue[mapperRequest](groupCtx, c.ctrl) // Watched Type Events → Mapper Queue group.Go(func() error { - return c.watch(groupCtx, watch.watchedType, mapQueue.Add) + return c.watch(groupCtx, watch.watchedType, func(res *pbresource.Resource) { + mapQueue.Add(mapperRequest{res: res}) + }) }) // Mapper Queue → Mapper → Reconciliation Queue @@ -96,13 +98,13 @@ func (c *controllerRunner) watch(ctx context.Context, typ *pbresource.Type, add func (c *controllerRunner) runMapper( ctx context.Context, w watch, - from queue.WorkQueue[*pbresource.Resource], + from queue.WorkQueue[mapperRequest], to queue.WorkQueue[Request], ) error { logger := c.logger.With("watched_resource_type", resource.ToGVK(w.watchedType)) for { - res, shutdown := from.Get() + item, shutdown := from.Get() if shutdown { return nil } @@ -110,12 +112,12 @@ func (c *controllerRunner) runMapper( var reqs []Request err := c.handlePanic(func() error { var err error - reqs, err = w.mapper(ctx, c.runtime(), res) + reqs, err = w.mapper(ctx, c.runtime(), item.res) return err }) if err != nil { - from.AddRateLimited(res) - from.Done(res) + from.AddRateLimited(item) + from.Done(item) continue } @@ -130,8 +132,8 @@ func (c *controllerRunner) runMapper( to.Add(r) } - from.Forget(res) - from.Done(res) + from.Forget(item) + from.Done(item) } } @@ -183,3 +185,19 @@ func (c *controllerRunner) runtime() Runtime { Logger: c.logger, } } + +type mapperRequest struct{ res *pbresource.Resource } + +// Key satisfies the queue.ItemType interface. It returns a string which will be +// used to de-duplicate requests in the queue. +func (i mapperRequest) Key() string { + return fmt.Sprintf( + "type=%q,part=%q,peer=%q,ns=%q,name=%q,uid=%q", + resource.ToGVK(i.res.Id.Type), + i.res.Id.Tenancy.Partition, + i.res.Id.Tenancy.PeerName, + i.res.Id.Tenancy.Namespace, + i.res.Id.Name, + i.res.Id.Uid, + ) +} From 4f6da20fe51c3d58dd4ec503c95512727169ffff Mon Sep 17 00:00:00 2001 From: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Date: Tue, 9 May 2023 12:37:58 -0500 Subject: [PATCH 04/13] Fix multiple issues related to proxycfg health queries. (#17241) Fix multiple issues related to proxycfg health queries. 1. The datacenter was not being provided to a proxycfg query, which resulted in bypassing agentless query optimizations and using the normal API instead. 2. The health rpc endpoint would return a zero index when insufficient ACLs were detected. This would result in the agent cache performing an infinite loop of queries in rapid succession without backoff. --- .changelog/17241.txt | 3 +++ agent/consul/health_endpoint.go | 37 ++++++++++++++-------------- agent/consul/health_endpoint_test.go | 1 + agent/proxycfg/state_test.go | 2 +- agent/proxycfg/upstreams.go | 24 +++++++++--------- 5 files changed, 37 insertions(+), 30 deletions(-) create mode 100644 .changelog/17241.txt diff --git a/.changelog/17241.txt b/.changelog/17241.txt new file mode 100644 index 000000000000..0369710928ed --- /dev/null +++ b/.changelog/17241.txt @@ -0,0 +1,3 @@ +```release-note:bug +connect: Fix multiple inefficient behaviors when querying service health. +``` diff --git a/agent/consul/health_endpoint.go b/agent/consul/health_endpoint.go index abf17adbaeb1..6913136d3844 100644 --- a/agent/consul/health_endpoint.go +++ b/agent/consul/health_endpoint.go @@ -214,28 +214,10 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc f = h.serviceNodesDefault } - authzContext := acl.AuthorizerContext{ - Peer: args.PeerName, - } - authz, err := h.srv.ResolveTokenAndDefaultMeta(args.Token, &args.EnterpriseMeta, &authzContext) - if err != nil { - return err - } - if err := h.srv.validateEnterpriseRequest(&args.EnterpriseMeta, false); err != nil { return err } - // If we're doing a connect or ingress query, we need read access to the service - // we're trying to find proxies for, so check that. - if args.Connect || args.Ingress { - // TODO(acl-error-enhancements) Look for ways to percolate this information up to give any feedback to the user. - if authz.ServiceRead(args.ServiceName, &authzContext) != acl.Allow { - // Just return nil, which will return an empty response (tested) - return nil - } - } - filter, err := bexpr.CreateFilter(args.Filter, nil, reply.Nodes) if err != nil { return err @@ -257,6 +239,25 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc return err } + authzContext := acl.AuthorizerContext{ + Peer: args.PeerName, + } + authz, err := h.srv.ResolveTokenAndDefaultMeta(args.Token, &args.EnterpriseMeta, &authzContext) + if err != nil { + return err + } + + // If we're doing a connect or ingress query, we need read access to the service + // we're trying to find proxies for, so check that. + if args.Connect || args.Ingress { + // TODO(acl-error-enhancements) Look for ways to percolate this information up to give any feedback to the user. + if authz.ServiceRead(args.ServiceName, &authzContext) != acl.Allow { + // Return the index here so that the agent cache does not infinitely loop. + reply.Index = index + return nil + } + } + resolvedNodes := nodes if args.MergeCentralConfig { for _, node := range resolvedNodes { diff --git a/agent/consul/health_endpoint_test.go b/agent/consul/health_endpoint_test.go index 4b492081c930..1981332d86b4 100644 --- a/agent/consul/health_endpoint_test.go +++ b/agent/consul/health_endpoint_test.go @@ -1127,6 +1127,7 @@ node "foo" { var resp structs.IndexedCheckServiceNodes assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Health.ServiceNodes", &req, &resp)) assert.Len(t, resp.Nodes, 0) + assert.Greater(t, resp.Index, uint64(0)) // List w/ token. This should work since we're requesting "foo", but should // also only contain the proxies with names that adhere to our ACL. diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index aa11dc43f542..792367b473be 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -795,7 +795,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { fmt.Sprintf("upstream-target:api-failover-remote.default.default.dc2:%s-failover-remote?dc=dc2", apiUID.String()): genVerifyServiceSpecificRequest("api-failover-remote", "", "dc2", true), fmt.Sprintf("upstream-target:api-failover-local.default.default.dc2:%s-failover-local?dc=dc2", apiUID.String()): genVerifyServiceSpecificRequest("api-failover-local", "", "dc2", true), fmt.Sprintf("upstream-target:api-failover-direct.default.default.dc2:%s-failover-direct?dc=dc2", apiUID.String()): genVerifyServiceSpecificRequest("api-failover-direct", "", "dc2", true), - upstreamPeerWatchIDPrefix + fmt.Sprintf("%s-failover-to-peer?peer=cluster-01", apiUID.String()): genVerifyServiceSpecificPeeredRequest("api-failover-to-peer", "", "", "cluster-01", true), + upstreamPeerWatchIDPrefix + fmt.Sprintf("%s-failover-to-peer?peer=cluster-01", apiUID.String()): genVerifyServiceSpecificPeeredRequest("api-failover-to-peer", "", "dc1", "cluster-01", true), fmt.Sprintf("mesh-gateway:dc2:%s-failover-remote?dc=dc2", apiUID.String()): genVerifyGatewayWatch("dc2"), fmt.Sprintf("mesh-gateway:dc1:%s-failover-local?dc=dc2", apiUID.String()): genVerifyGatewayWatch("dc1"), }, diff --git a/agent/proxycfg/upstreams.go b/agent/proxycfg/upstreams.go index d7fee4a3fcc1..5e42072fac32 100644 --- a/agent/proxycfg/upstreams.go +++ b/agent/proxycfg/upstreams.go @@ -316,8 +316,19 @@ func (s *handlerUpstreams) resetWatchesFromChain( watchedChainEndpoints = true } - opts := targetWatchOpts{upstreamID: uid} - opts.fromChainTarget(target) + opts := targetWatchOpts{ + upstreamID: uid, + chainID: target.ID, + service: target.Service, + filter: target.Subset.Filter, + datacenter: target.Datacenter, + peer: target.Peer, + entMeta: target.GetEnterpriseMetadata(), + } + // Peering targets do not set the datacenter field, so we should default it here. + if opts.datacenter == "" { + opts.datacenter = s.source.Datacenter + } err := s.watchUpstreamTarget(ctx, snap, opts) if err != nil { @@ -435,15 +446,6 @@ type targetWatchOpts struct { entMeta *acl.EnterpriseMeta } -func (o *targetWatchOpts) fromChainTarget(t *structs.DiscoveryTarget) { - o.chainID = t.ID - o.service = t.Service - o.filter = t.Subset.Filter - o.datacenter = t.Datacenter - o.peer = t.Peer - o.entMeta = t.GetEnterpriseMetadata() -} - func (s *handlerUpstreams) watchUpstreamTarget(ctx context.Context, snap *ConfigSnapshotUpstreams, opts targetWatchOpts) error { s.logger.Trace("initializing watch of target", "upstream", opts.upstreamID, From d53a1d4a27ba22071ac472a05a174008afb81fbb Mon Sep 17 00:00:00 2001 From: Dan Upton Date: Tue, 9 May 2023 19:02:24 +0100 Subject: [PATCH 05/13] resource: add helpers for more efficiently comparing IDs etc (#17224) --- .../grpc-external/services/resource/delete.go | 3 +- .../grpc-external/services/resource/write.go | 5 +- internal/controller/controller.go | 3 +- internal/resource/demo/controller.go | 2 +- internal/resource/equality.go | 150 +++++ internal/resource/equality_test.go | 629 ++++++++++++++++++ internal/resource/status.go | 46 -- internal/resource/status_test.go | 129 ---- 8 files changed, 784 insertions(+), 183 deletions(-) create mode 100644 internal/resource/equality.go create mode 100644 internal/resource/equality_test.go delete mode 100644 internal/resource/status.go delete mode 100644 internal/resource/status_test.go diff --git a/agent/grpc-external/services/resource/delete.go b/agent/grpc-external/services/resource/delete.go index f15fafa5931e..b3045b3d6d29 100644 --- a/agent/grpc-external/services/resource/delete.go +++ b/agent/grpc-external/services/resource/delete.go @@ -12,7 +12,6 @@ import ( "github.com/oklog/ulid/v2" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "github.com/hashicorp/consul/acl" @@ -95,7 +94,7 @@ func (s *Server) Delete(ctx context.Context, req *pbresource.DeleteRequest) (*pb // still be deleted from the system by the reaper controller. func (s *Server) maybeCreateTombstone(ctx context.Context, deleteId *pbresource.ID) error { // Don't create a tombstone when the resource being deleted is itself a tombstone. - if proto.Equal(resource.TypeV1Tombstone, deleteId.Type) { + if resource.EqualType(resource.TypeV1Tombstone, deleteId.Type) { return nil } diff --git a/agent/grpc-external/services/resource/write.go b/agent/grpc-external/services/resource/write.go index ad17e61c51d9..d50ca1dd8bcd 100644 --- a/agent/grpc-external/services/resource/write.go +++ b/agent/grpc-external/services/resource/write.go @@ -12,7 +12,6 @@ import ( "github.com/oklog/ulid/v2" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/internal/resource" @@ -176,14 +175,14 @@ func (s *Server) Write(ctx context.Context, req *pbresource.WriteRequest) (*pbre } // Owner can only be set on creation. Enforce immutability. - if !proto.Equal(input.Owner, existing.Owner) { + if !resource.EqualID(input.Owner, existing.Owner) { return status.Errorf(codes.InvalidArgument, "owner cannot be changed") } // Carry over status and prevent updates if input.Status == nil { input.Status = existing.Status - } else if !resource.EqualStatus(input.Status, existing.Status) { + } else if !resource.EqualStatusMap(input.Status, existing.Status) { return errUseWriteStatus } diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 296ff5faf4f5..54d5c57386a3 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -11,7 +11,6 @@ import ( "github.com/hashicorp/go-hclog" "golang.org/x/sync/errgroup" - "google.golang.org/protobuf/proto" "github.com/hashicorp/consul/agent/consul/controller/queue" "github.com/hashicorp/consul/internal/resource" @@ -122,7 +121,7 @@ func (c *controllerRunner) runMapper( } for _, r := range reqs { - if !proto.Equal(r.ID.Type, c.ctrl.managedType) { + if !resource.EqualType(r.ID.Type, c.ctrl.managedType) { logger.Error("dependency mapper returned request for a resource of the wrong type", "type_expected", resource.ToGVK(c.ctrl.managedType), "type_got", resource.ToGVK(r.ID.Type), diff --git a/internal/resource/demo/controller.go b/internal/resource/demo/controller.go index 11de1c5057ee..db935ec065ba 100644 --- a/internal/resource/demo/controller.go +++ b/internal/resource/demo/controller.go @@ -111,7 +111,7 @@ func (r *artistReconciler) Reconcile(ctx context.Context, rt controller.Runtime, Conditions: conditions, } - if proto.Equal(res.Status[statusKeyArtistController], newStatus) { + if resource.EqualStatus(res.Status[statusKeyArtistController], newStatus) { return nil } diff --git a/internal/resource/equality.go b/internal/resource/equality.go new file mode 100644 index 000000000000..100d4c7c95e2 --- /dev/null +++ b/internal/resource/equality.go @@ -0,0 +1,150 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package resource + +import "github.com/hashicorp/consul/proto-public/pbresource" + +// EqualType compares two resource types for equality without reflection. +func EqualType(a, b *pbresource.Type) bool { + if a == b { + return true + } + + if a == nil || b == nil { + return false + } + + return a.Group == b.Group && + a.GroupVersion == b.GroupVersion && + a.Kind == b.Kind +} + +// EqualType compares two resource tenancies for equality without reflection. +func EqualTenancy(a, b *pbresource.Tenancy) bool { + if a == b { + return true + } + + if a == nil || b == nil { + return false + } + + return a.Partition == b.Partition && + a.PeerName == b.PeerName && + a.Namespace == b.Namespace +} + +// EqualType compares two resource IDs for equality without reflection. +func EqualID(a, b *pbresource.ID) bool { + if a == b { + return true + } + + if a == nil || b == nil { + return false + } + + return EqualType(a.Type, b.Type) && + EqualTenancy(a.Tenancy, b.Tenancy) && + a.Name == b.Name && + a.Uid == b.Uid +} + +// EqualStatus compares two statuses for equality without reflection. +func EqualStatus(a, b *pbresource.Status) bool { + if a == b { + return true + } + + if a == nil || b == nil { + return false + } + + if a.ObservedGeneration != b.ObservedGeneration { + return false + } + + if len(a.Conditions) != len(b.Conditions) { + return false + } + + for i, ac := range a.Conditions { + bc := b.Conditions[i] + + if !EqualCondition(ac, bc) { + return false + } + } + + return true +} + +// EqualCondition compares two conditions for equality without reflection. +func EqualCondition(a, b *pbresource.Condition) bool { + if a == b { + return true + } + + if a == nil || b == nil { + return false + } + + return a.Type == b.Type && + a.State == b.State && + a.Reason == b.Reason && + a.Message == b.Message && + EqualReference(a.Resource, b.Resource) +} + +// EqualReference compares two references for equality without reflection. +func EqualReference(a, b *pbresource.Reference) bool { + if a == b { + return true + } + + if a == nil || b == nil { + return false + } + + return EqualType(a.Type, b.Type) && + EqualTenancy(a.Tenancy, b.Tenancy) && + a.Name == b.Name && + a.Section == b.Section +} + +// EqualStatusMap compares two status maps for equality without reflection. +func EqualStatusMap(a, b map[string]*pbresource.Status) bool { + if len(a) != len(b) { + return false + } + + compared := make(map[string]struct{}) + for k, av := range a { + bv, ok := b[k] + if !ok { + return false + } + if !EqualStatus(av, bv) { + return false + } + compared[k] = struct{}{} + } + + for k, bv := range b { + if _, skip := compared[k]; skip { + continue + } + + av, ok := a[k] + if !ok { + return false + } + + if !EqualStatus(av, bv) { + return false + } + } + + return true +} diff --git a/internal/resource/equality_test.go b/internal/resource/equality_test.go new file mode 100644 index 000000000000..45af9ea1801b --- /dev/null +++ b/internal/resource/equality_test.go @@ -0,0 +1,629 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package resource_test + +import ( + "fmt" + "testing" + + "github.com/oklog/ulid/v2" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/hashicorp/consul/internal/resource" + "github.com/hashicorp/consul/proto-public/pbresource" +) + +func TestEqualType(t *testing.T) { + t.Run("same pointer", func(t *testing.T) { + typ := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "bar", + } + require.True(t, resource.EqualType(typ, typ)) + }) + + t.Run("equal", func(t *testing.T) { + a := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "bar", + } + b := clone(a) + require.True(t, resource.EqualType(a, b)) + }) + + t.Run("nil", func(t *testing.T) { + a := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "bar", + } + require.False(t, resource.EqualType(a, nil)) + require.False(t, resource.EqualType(nil, a)) + }) + + t.Run("different Group", func(t *testing.T) { + a := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "bar", + } + b := clone(a) + b.Group = "bar" + require.False(t, resource.EqualType(a, b)) + }) + + t.Run("different GroupVersion", func(t *testing.T) { + a := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "bar", + } + b := clone(a) + b.GroupVersion = "v2" + require.False(t, resource.EqualType(a, b)) + }) + + t.Run("different Kind", func(t *testing.T) { + a := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "bar", + } + b := clone(a) + b.Kind = "baz" + require.False(t, resource.EqualType(a, b)) + }) +} + +func TestEqualTenancy(t *testing.T) { + t.Run("same pointer", func(t *testing.T) { + ten := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + } + require.True(t, resource.EqualTenancy(ten, ten)) + }) + + t.Run("equal", func(t *testing.T) { + a := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + } + b := clone(a) + require.True(t, resource.EqualTenancy(a, b)) + }) + + t.Run("nil", func(t *testing.T) { + a := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + } + require.False(t, resource.EqualTenancy(a, nil)) + require.False(t, resource.EqualTenancy(nil, a)) + }) + + t.Run("different Partition", func(t *testing.T) { + a := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + } + b := clone(a) + b.Partition = "qux" + require.False(t, resource.EqualTenancy(a, b)) + }) + + t.Run("different PeerName", func(t *testing.T) { + a := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + } + b := clone(a) + b.PeerName = "qux" + require.False(t, resource.EqualTenancy(a, b)) + }) + + t.Run("different Namespace", func(t *testing.T) { + a := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + } + b := clone(a) + b.Namespace = "qux" + require.False(t, resource.EqualTenancy(a, b)) + }) +} + +func TestEqualID(t *testing.T) { + t.Run("same pointer", func(t *testing.T) { + id := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + require.True(t, resource.EqualID(id, id)) + }) + + t.Run("equal", func(t *testing.T) { + a := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + b := clone(a) + require.True(t, resource.EqualID(a, b)) + }) + + t.Run("nil", func(t *testing.T) { + a := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + require.False(t, resource.EqualID(a, nil)) + require.False(t, resource.EqualID(nil, a)) + }) + + t.Run("different type", func(t *testing.T) { + a := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + b := clone(a) + b.Type.Kind = "album" + require.False(t, resource.EqualID(a, b)) + }) + + t.Run("different tenancy", func(t *testing.T) { + a := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + b := clone(a) + b.Tenancy.Namespace = "qux" + require.False(t, resource.EqualID(a, b)) + }) + + t.Run("different name", func(t *testing.T) { + a := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + b := clone(a) + b.Name = "boom" + require.False(t, resource.EqualID(a, b)) + }) + + t.Run("different uid", func(t *testing.T) { + a := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + b := clone(a) + b.Uid = ulid.Make().String() + require.False(t, resource.EqualID(a, b)) + }) +} + +func TestEqualStatus(t *testing.T) { + orig := &pbresource.Status{ + ObservedGeneration: ulid.Make().String(), + Conditions: []*pbresource.Condition{ + { + Type: "FooType", + State: pbresource.Condition_STATE_TRUE, + Reason: "FooReason", + Message: "Foo is true", + Resource: &pbresource.Reference{ + Type: &pbresource.Type{ + Group: "foo-group", + GroupVersion: "foo-group-version", + Kind: "foo-kind", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo-partition", + PeerName: "foo-peer-name", + Namespace: "foo-namespace", + }, + Name: "foo-name", + Section: "foo-section", + }, + }, + }, + } + + // Equal cases. + t.Run("same pointer", func(t *testing.T) { + require.True(t, resource.EqualStatus(orig, orig)) + }) + + t.Run("equal", func(t *testing.T) { + require.True(t, resource.EqualStatus(orig, clone(orig))) + }) + + // Not equal cases. + t.Run("nil", func(t *testing.T) { + require.False(t, resource.EqualStatus(orig, nil)) + require.False(t, resource.EqualStatus(nil, orig)) + }) + + testCases := map[string]func(*pbresource.Status){ + "different ObservedGeneration": func(s *pbresource.Status) { + s.ObservedGeneration = "" + }, + "different Conditions": func(s *pbresource.Status) { + s.Conditions = append(s.Conditions, s.Conditions...) + }, + "nil Condition": func(s *pbresource.Status) { + s.Conditions[0] = nil + }, + "different Condition.Type": func(s *pbresource.Status) { + s.Conditions[0].Type = "BarType" + }, + "different Condition.State": func(s *pbresource.Status) { + s.Conditions[0].State = pbresource.Condition_STATE_FALSE + }, + "different Condition.Reason": func(s *pbresource.Status) { + s.Conditions[0].Reason = "BarReason" + }, + "different Condition.Message": func(s *pbresource.Status) { + s.Conditions[0].Reason = "Bar if false" + }, + "different Condition.Resource": func(s *pbresource.Status) { + s.Conditions[0].Resource = nil + }, + "different Condition.Resource.Type": func(s *pbresource.Status) { + s.Conditions[0].Resource.Type.Group = "bar-group" + }, + "different Condition.Resource.Tenancy": func(s *pbresource.Status) { + s.Conditions[0].Resource.Tenancy.Partition = "bar-partition" + }, + "different Condition.Resource.Name": func(s *pbresource.Status) { + s.Conditions[0].Resource.Name = "bar-name" + }, + "different Condition.Resource.Section": func(s *pbresource.Status) { + s.Conditions[0].Resource.Section = "bar-section" + }, + } + for desc, modFn := range testCases { + t.Run(desc, func(t *testing.T) { + a, b := clone(orig), clone(orig) + modFn(b) + + require.False(t, resource.EqualStatus(a, b)) + require.False(t, resource.EqualStatus(b, a)) + }) + } +} + +func TestEqualStatusMap(t *testing.T) { + generation := ulid.Make().String() + + for idx, tc := range []struct { + a, b map[string]*pbresource.Status + equal bool + }{ + {nil, nil, true}, + {nil, map[string]*pbresource.Status{}, true}, + { + map[string]*pbresource.Status{ + "consul.io/some-controller": { + ObservedGeneration: generation, + Conditions: []*pbresource.Condition{ + { + Type: "Foo", + State: pbresource.Condition_STATE_TRUE, + Reason: "Bar", + Message: "Foo is true because of Bar", + }, + }, + }, + }, + map[string]*pbresource.Status{ + "consul.io/some-controller": { + ObservedGeneration: generation, + Conditions: []*pbresource.Condition{ + { + Type: "Foo", + State: pbresource.Condition_STATE_TRUE, + Reason: "Bar", + Message: "Foo is true because of Bar", + }, + }, + }, + }, + true, + }, + { + map[string]*pbresource.Status{ + "consul.io/some-controller": { + ObservedGeneration: generation, + Conditions: []*pbresource.Condition{ + { + Type: "Foo", + State: pbresource.Condition_STATE_TRUE, + Reason: "Bar", + Message: "Foo is true because of Bar", + }, + }, + }, + }, + map[string]*pbresource.Status{ + "consul.io/some-controller": { + ObservedGeneration: generation, + Conditions: []*pbresource.Condition{ + { + Type: "Foo", + State: pbresource.Condition_STATE_FALSE, + Reason: "Bar", + Message: "Foo is false because of Bar", + }, + }, + }, + }, + false, + }, + { + map[string]*pbresource.Status{ + "consul.io/some-controller": { + ObservedGeneration: generation, + Conditions: []*pbresource.Condition{ + { + Type: "Foo", + State: pbresource.Condition_STATE_TRUE, + Reason: "Bar", + Message: "Foo is true because of Bar", + }, + }, + }, + }, + map[string]*pbresource.Status{ + "consul.io/some-controller": { + ObservedGeneration: generation, + Conditions: []*pbresource.Condition{ + { + Type: "Foo", + State: pbresource.Condition_STATE_TRUE, + Reason: "Bar", + Message: "Foo is true because of Bar", + }, + }, + }, + "consul.io/other-controller": { + ObservedGeneration: generation, + Conditions: []*pbresource.Condition{ + { + Type: "Foo", + State: pbresource.Condition_STATE_TRUE, + Reason: "Bar", + Message: "Foo is true because of Bar", + }, + }, + }, + }, + false, + }, + } { + t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { + require.Equal(t, tc.equal, resource.EqualStatusMap(tc.a, tc.b)) + require.Equal(t, tc.equal, resource.EqualStatusMap(tc.b, tc.a)) + }) + } +} + +func BenchmarkEqualType(b *testing.B) { + // cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz + // BenchmarkEqualType/ours-16 161532109 7.309 ns/op 0 B/op 0 allocs/op + // BenchmarkEqualType/reflection-16 1584954 748.4 ns/op 160 B/op 9 allocs/op + typeA := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "bar", + } + typeB := &pbresource.Type{ + Group: "foo", + GroupVersion: "v1", + Kind: "baz", + } + b.ResetTimer() + + b.Run("ours", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = resource.EqualType(typeA, typeB) + } + }) + + b.Run("reflection", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = proto.Equal(typeA, typeB) + } + }) +} + +func BenchmarkEqualTenancy(b *testing.B) { + // cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz + // BenchmarkEqualTenancy/ours-16 159998534 7.426 ns/op 0 B/op 0 allocs/op + // BenchmarkEqualTenancy/reflection-16 2283500 550.3 ns/op 128 B/op 7 allocs/op + tenA := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + } + tenB := &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "qux", + } + b.ResetTimer() + + b.Run("ours", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = resource.EqualTenancy(tenA, tenB) + } + }) + + b.Run("reflection", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = proto.Equal(tenA, tenB) + } + }) +} + +func BenchmarkEqualID(b *testing.B) { + // cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz + // BenchmarkEqualID/ours-16 57818125 21.40 ns/op 0 B/op 0 allocs/op + // BenchmarkEqualID/reflection-16 3596365 330.1 ns/op 96 B/op 5 allocs/op + idA := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "demo", + GroupVersion: "v2", + Kind: "artist", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo", + PeerName: "bar", + Namespace: "baz", + }, + Name: "qux", + Uid: ulid.Make().String(), + } + idB := clone(idA) + idB.Uid = ulid.Make().String() + b.ResetTimer() + + b.Run("ours", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = resource.EqualID(idA, idB) + } + }) + + b.Run("reflection", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = proto.Equal(idA, idB) + } + }) +} + +func BenchmarkEqualStatus(b *testing.B) { + // cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz + // BenchmarkEqualStatus/ours-16 38648232 30.75 ns/op 0 B/op 0 allocs/op + // BenchmarkEqualStatus/reflection-16 237694 5267 ns/op 944 B/op 51 allocs/op + statusA := &pbresource.Status{ + ObservedGeneration: ulid.Make().String(), + Conditions: []*pbresource.Condition{ + { + Type: "FooType", + State: pbresource.Condition_STATE_TRUE, + Reason: "FooReason", + Message: "Foo is true", + Resource: &pbresource.Reference{ + Type: &pbresource.Type{ + Group: "foo-group", + GroupVersion: "foo-group-version", + Kind: "foo-kind", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "foo-partition", + PeerName: "foo-peer-name", + Namespace: "foo-namespace", + }, + Name: "foo-name", + Section: "foo-section", + }, + }, + }, + } + statusB := clone(statusA) + statusB.Conditions[0].Resource.Section = "bar-section" + b.ResetTimer() + + b.Run("ours", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = resource.EqualStatus(statusA, statusB) + } + }) + + b.Run("reflection", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = proto.Equal(statusA, statusB) + } + }) +} + +func clone[T proto.Message](v T) T { return proto.Clone(v).(T) } diff --git a/internal/resource/status.go b/internal/resource/status.go deleted file mode 100644 index 89979b9f6080..000000000000 --- a/internal/resource/status.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package resource - -import ( - "google.golang.org/protobuf/proto" - - "github.com/hashicorp/consul/proto-public/pbresource" -) - -// EqualStatus compares two status maps for equality. -func EqualStatus(a, b map[string]*pbresource.Status) bool { - if len(a) != len(b) { - return false - } - - compared := make(map[string]struct{}) - for k, av := range a { - bv, ok := b[k] - if !ok { - return false - } - if !proto.Equal(av, bv) { - return false - } - compared[k] = struct{}{} - } - - for k, bv := range b { - if _, skip := compared[k]; skip { - continue - } - - av, ok := a[k] - if !ok { - return false - } - - if !proto.Equal(av, bv) { - return false - } - } - - return true -} diff --git a/internal/resource/status_test.go b/internal/resource/status_test.go deleted file mode 100644 index 39d69ff03f2f..000000000000 --- a/internal/resource/status_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package resource - -import ( - "fmt" - "testing" - - "github.com/oklog/ulid/v2" - "github.com/stretchr/testify/require" - - "github.com/hashicorp/consul/proto-public/pbresource" -) - -func TestEqualStatus(t *testing.T) { - generation := ulid.Make().String() - - for idx, tc := range []struct { - a, b map[string]*pbresource.Status - equal bool - }{ - {nil, nil, true}, - {nil, map[string]*pbresource.Status{}, true}, - { - map[string]*pbresource.Status{ - "consul.io/some-controller": { - ObservedGeneration: generation, - Conditions: []*pbresource.Condition{ - { - Type: "Foo", - State: pbresource.Condition_STATE_TRUE, - Reason: "Bar", - Message: "Foo is true because of Bar", - }, - }, - }, - }, - map[string]*pbresource.Status{ - "consul.io/some-controller": { - ObservedGeneration: generation, - Conditions: []*pbresource.Condition{ - { - Type: "Foo", - State: pbresource.Condition_STATE_TRUE, - Reason: "Bar", - Message: "Foo is true because of Bar", - }, - }, - }, - }, - true, - }, - { - map[string]*pbresource.Status{ - "consul.io/some-controller": { - ObservedGeneration: generation, - Conditions: []*pbresource.Condition{ - { - Type: "Foo", - State: pbresource.Condition_STATE_TRUE, - Reason: "Bar", - Message: "Foo is true because of Bar", - }, - }, - }, - }, - map[string]*pbresource.Status{ - "consul.io/some-controller": { - ObservedGeneration: generation, - Conditions: []*pbresource.Condition{ - { - Type: "Foo", - State: pbresource.Condition_STATE_FALSE, - Reason: "Bar", - Message: "Foo is false because of Bar", - }, - }, - }, - }, - false, - }, - { - map[string]*pbresource.Status{ - "consul.io/some-controller": { - ObservedGeneration: generation, - Conditions: []*pbresource.Condition{ - { - Type: "Foo", - State: pbresource.Condition_STATE_TRUE, - Reason: "Bar", - Message: "Foo is true because of Bar", - }, - }, - }, - }, - map[string]*pbresource.Status{ - "consul.io/some-controller": { - ObservedGeneration: generation, - Conditions: []*pbresource.Condition{ - { - Type: "Foo", - State: pbresource.Condition_STATE_TRUE, - Reason: "Bar", - Message: "Foo is true because of Bar", - }, - }, - }, - "consul.io/other-controller": { - ObservedGeneration: generation, - Conditions: []*pbresource.Condition{ - { - Type: "Foo", - State: pbresource.Condition_STATE_TRUE, - Reason: "Bar", - Message: "Foo is true because of Bar", - }, - }, - }, - }, - false, - }, - } { - t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { - require.Equal(t, tc.equal, EqualStatus(tc.a, tc.b)) - require.Equal(t, tc.equal, EqualStatus(tc.b, tc.a)) - }) - } -} From 7c3e9cd86211e8e513198ce00c5f5644cbc54844 Mon Sep 17 00:00:00 2001 From: Freddy Date: Tue, 9 May 2023 12:20:26 -0600 Subject: [PATCH 06/13] Hash namespace+proxy ID when creating socket path (#17204) UNIX domain socket paths are limited to 104-108 characters, depending on the OS. This limit was quite easy to exceed when testing the feature on Kubernetes, due to how proxy IDs encode the Pod ID eg: metrics-collector-59467bcb9b-fkkzl-hcp-metrics-collector-sidecar-proxy To ensure we stay under that character limit this commit makes a couple changes: - Use a b64 encoded SHA1 hash of the namespace + proxy ID to create a short and deterministic socket file name. - Add validation to proxy registrations and proxy-defaults to enforce a limit on the socket directory length. --- agent/proxycfg/connect_proxy.go | 12 +- agent/proxycfg/state_test.go | 2 +- agent/structs/config_entry.go | 15 ++ agent/structs/config_entry_test.go | 39 ++++ agent/structs/structs.go | 4 + agent/structs/structs_test.go | 10 + .../listeners/hcp-metrics.latest.golden | 204 +++++++++--------- command/connect/envoy/bootstrap_config.go | 26 ++- .../connect/envoy/bootstrap_config_test.go | 4 +- .../connect/envoy/testdata/hcp-metrics.golden | 2 +- 10 files changed, 205 insertions(+), 113 deletions(-) diff --git a/agent/proxycfg/connect_proxy.go b/agent/proxycfg/connect_proxy.go index f9c46103a094..873f942ba407 100644 --- a/agent/proxycfg/connect_proxy.go +++ b/agent/proxycfg/connect_proxy.go @@ -5,6 +5,8 @@ package proxycfg import ( "context" + "crypto/sha1" + "encoding/base64" "fmt" "path" "strings" @@ -660,8 +662,14 @@ func (s *handlerConnectProxy) maybeInitializeHCPMetricsWatches(ctx context.Conte // The path includes the proxy ID so that when multiple proxies are on the same host // they each have a distinct path to send their metrics. - sock := fmt.Sprintf("%s_%s.sock", s.proxyID.NamespaceOrDefault(), s.proxyID.ID) - path := path.Join(hcpCfg.HCPMetricsBindSocketDir, sock) + id := s.proxyID.NamespaceOrDefault() + "_" + s.proxyID.ID + + // UNIX domain sockets paths have a max length of 108, so we take a hash of the compound ID + // to limit the length of the socket path. + h := sha1.New() + h.Write([]byte(id)) + hash := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) + path := path.Join(hcpCfg.HCPMetricsBindSocketDir, hash+".sock") upstream := structs.Upstream{ DestinationNamespace: acl.DefaultNamespaceName, diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 792367b473be..f8760a5e11a3 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -3722,7 +3722,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { DestinationNamespace: "default", DestinationPartition: "default", DestinationName: apimod.HCPMetricsCollectorName, - LocalBindSocketPath: "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock", + LocalBindSocketPath: "/tmp/consul/hcp-metrics/gqmuzdHCUPAEY5mbF8vgkZCNI14.sock", Config: map[string]interface{}{ "protocol": "grpc", }, diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index dd6671268e0d..c18a8013b6d4 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -468,6 +468,10 @@ func (e *ProxyConfigEntry) Validate() error { return err } + if err := validateOpaqueProxyConfig(e.Config); err != nil { + return fmt.Errorf("Config: %w", err) + } + if err := envoyextensions.ValidateExtensions(e.EnvoyExtensions.ToAPI()); err != nil { return err } @@ -1329,6 +1333,17 @@ func (c *ConfigEntryResponse) UnmarshalBinary(data []byte) error { return nil } +func validateOpaqueProxyConfig(config map[string]interface{}) error { + // This max is chosen to stay under the 104 character limit on OpenBSD, FreeBSD, MacOS. + // It assumes the socket's filename is fixed at 32 characters. + const maxSocketDirLen = 70 + + if path, _ := config["envoy_hcp_metrics_bind_socket_dir"].(string); len(path) > maxSocketDirLen { + return fmt.Errorf("envoy_hcp_metrics_bind_socket_dir length %d exceeds max %d", len(path), maxSocketDirLen) + } + return nil +} + func validateConfigEntryMeta(meta map[string]string) error { var err error if len(meta) > metaMaxKeyPairs { diff --git a/agent/structs/config_entry_test.go b/agent/structs/config_entry_test.go index b7fad39a37a5..9143dc05d993 100644 --- a/agent/structs/config_entry_test.go +++ b/agent/structs/config_entry_test.go @@ -3261,6 +3261,15 @@ func TestProxyConfigEntry(t *testing.T) { EnterpriseMeta: *acl.DefaultEnterpriseMeta(), }, }, + "proxy config entry has invalid opaque config": { + entry: &ProxyConfigEntry{ + Name: "global", + Config: map[string]interface{}{ + "envoy_hcp_metrics_bind_socket_dir": "/Consul/is/a/networking/platform/that/enables/securing/your/networking/", + }, + }, + validateErr: "Config: envoy_hcp_metrics_bind_socket_dir length 71 exceeds max", + }, "proxy config has invalid failover policy": { entry: &ProxyConfigEntry{ Name: "global", @@ -3445,3 +3454,33 @@ func uintPointer(v uint32) *uint32 { func durationPointer(d time.Duration) *time.Duration { return &d } + +func TestValidateOpaqueConfigMap(t *testing.T) { + tt := map[string]struct { + input map[string]interface{} + expectErr string + }{ + "hcp metrics socket dir is valid": { + input: map[string]interface{}{ + "envoy_hcp_metrics_bind_socket_dir": "/etc/consul.d/hcp"}, + expectErr: "", + }, + "hcp metrics socket dir is too long": { + input: map[string]interface{}{ + "envoy_hcp_metrics_bind_socket_dir": "/Consul/is/a/networking/platform/that/enables/securing/your/networking/", + }, + expectErr: "envoy_hcp_metrics_bind_socket_dir length 71 exceeds max 70", + }, + } + + for name, tc := range tt { + t.Run(name, func(t *testing.T) { + err := validateOpaqueProxyConfig(tc.input) + if tc.expectErr != "" { + require.ErrorContains(t, err, tc.expectErr) + return + } + require.NoError(t, err) + }) + } +} diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 3e12999e7baf..1771b18be03b 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -1508,6 +1508,10 @@ func (s *NodeService) ValidateForAgent() error { "A Proxy cannot also be Connect Native, only typical services")) } + if err := validateOpaqueProxyConfig(s.Proxy.Config); err != nil { + result = multierror.Append(result, fmt.Errorf("Proxy.Config: %w", err)) + } + // ensure we don't have multiple upstreams for the same service var ( upstreamKeys = make(map[UpstreamKey]struct{}) diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go index 49861f86c29c..6d887da9ac77 100644 --- a/agent/structs/structs_test.go +++ b/agent/structs/structs_test.go @@ -826,6 +826,16 @@ func TestStructs_NodeService_ValidateConnectProxy(t *testing.T) { "", }, + { + "connect-proxy: invalid opaque config", + func(x *NodeService) { + x.Proxy.Config = map[string]interface{}{ + "envoy_hcp_metrics_bind_socket_dir": "/Consul/is/a/networking/platform/that/enables/securing/your/networking/", + } + }, + "Proxy.Config: envoy_hcp_metrics_bind_socket_dir length 71 exceeds max", + }, + { "connect-proxy: no Proxy.DestinationServiceName", func(x *NodeService) { x.Proxy.DestinationServiceName = "" }, diff --git a/agent/xds/testdata/listeners/hcp-metrics.latest.golden b/agent/xds/testdata/listeners/hcp-metrics.latest.golden index d89036a935cb..c1e0d04734a3 100644 --- a/agent/xds/testdata/listeners/hcp-metrics.latest.golden +++ b/agent/xds/testdata/listeners/hcp-metrics.latest.golden @@ -1,184 +1,184 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "db:127.0.0.1:9191", - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 9191 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.db.default.default.dc1", - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" } } ] } ], - "trafficDirection": "OUTBOUND" + "trafficDirection": "OUTBOUND" }, { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "hcp-metrics-collector:/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock", - "address": { - "pipe": { - "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "hcp-metrics-collector:/tmp/consul/hcp-metrics/gqmuzdHCUPAEY5mbF8vgkZCNI14.sock", + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/gqmuzdHCUPAEY5mbF8vgkZCNI14.sock" } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.http_connection_manager", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "statPrefix": "upstream.hcp-metrics-collector.default.default.dc1", - "routeConfig": { - "name": "hcp-metrics-collector", - "virtualHosts": [ + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "upstream.hcp-metrics-collector.default.default.dc1", + "routeConfig": { + "name": "hcp-metrics-collector", + "virtualHosts": [ { - "name": "hcp-metrics-collector.default.default.dc1", - "domains": [ + "name": "hcp-metrics-collector.default.default.dc1", + "domains": [ "*" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "hcp-metrics-collector.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "hcp-metrics-collector.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" } } ] } ] }, - "httpFilters": [ + "httpFilters": [ { - "name": "envoy.filters.http.grpc_stats", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig", - "statsForAllMethods": true + "name": "envoy.filters.http.grpc_stats", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig", + "statsForAllMethods": true } }, { - "name": "envoy.filters.http.grpc_http1_bridge", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_http1_bridge.v3.Config" + "name": "envoy.filters.http.grpc_http1_bridge", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_http1_bridge.v3.Config" } }, { - "name": "envoy.filters.http.router", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" } } ], - "tracing": { - "randomSampling": {} + "tracing": { + "randomSampling": {} }, - "http2ProtocolOptions": {} + "http2ProtocolOptions": {} } } ] } ], - "trafficDirection": "OUTBOUND" + "trafficDirection": "OUTBOUND" }, { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "prepared_query:geo-cache:127.10.10.10:8181", - "address": { - "socketAddress": { - "address": "127.10.10.10", - "portValue": 8181 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.prepared_query_geo-cache", - "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" } } ] } ], - "trafficDirection": "OUTBOUND" + "trafficDirection": "OUTBOUND" }, { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "public_listener:0.0.0.0:9999", - "address": { - "socketAddress": { - "address": "0.0.0.0", - "portValue": 9999 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.rbac", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", - "rules": {}, - "statPrefix": "connect_authz" + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": {}, + "statPrefix": "connect_authz" } }, { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "public_listener", - "cluster": "local_app" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" } } ], - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", - "commonTlsContext": { - "tlsParams": {}, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" } } }, - "requireClientCertificate": true + "requireClientCertificate": true } } } ], - "trafficDirection": "INBOUND" + "trafficDirection": "INBOUND" } ], - "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" } \ No newline at end of file diff --git a/command/connect/envoy/bootstrap_config.go b/command/connect/envoy/bootstrap_config.go index 337e1cf5db66..8ace2c37aa87 100644 --- a/command/connect/envoy/bootstrap_config.go +++ b/command/connect/envoy/bootstrap_config.go @@ -5,6 +5,8 @@ package envoy import ( "bytes" + "crypto/sha1" + "encoding/base64" "encoding/json" "fmt" "net" @@ -14,7 +16,6 @@ import ( "strings" "text/template" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/api" ) @@ -811,13 +812,28 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd return nil } -// appendHCPMetricsConfig generates config to enable a socket at path: /_.sock -// or /.sock, if namespace is empty. +// appendHCPMetricsConfig generates config to enable a socket at path: /.sock +// We take the hash of the compound proxy ID for a few reasons: +// +// - The proxy ID is included because this socket path must be unique per proxy. Each Envoy proxy will ship +// its metrics to HCP using its own loopback listener at this path. +// +// - The hash is needed because UNIX domain socket paths must be less than 104 characters. By using a b64 encoded +// SHA1 hash we end up with 27 chars for the name, 5 chars for the extension, and the remainder is saved for +// the configurable socket dir. The length of the directory's path is validated on writes to avoid going over. func appendHCPMetricsConfig(args *BootstrapTplArgs, hcpMetricsBindSocketDir string) { // Normalize namespace to "default". This ensures we match the namespace behaviour in proxycfg package, // where a dynamic listener will be created at the same socket path via xDS. - sock := fmt.Sprintf("%s_%s.sock", acl.NamespaceOrDefault(args.Namespace), args.ProxyID) - path := path.Join(hcpMetricsBindSocketDir, sock) + ns := args.Namespace + if ns == "" { + ns = "default" + } + id := ns + "_" + args.ProxyID + + h := sha1.New() + h.Write([]byte(id)) + hash := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) + path := path.Join(hcpMetricsBindSocketDir, hash+".sock") if args.StatsSinksJSON != "" { args.StatsSinksJSON += ",\n" diff --git a/command/connect/envoy/bootstrap_config_test.go b/command/connect/envoy/bootstrap_config_test.go index d9d10803c1d6..a43814c729a8 100644 --- a/command/connect/envoy/bootstrap_config_test.go +++ b/command/connect/envoy/bootstrap_config_test.go @@ -556,7 +556,7 @@ const ( "endpoint": { "address": { "pipe": { - "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + "path": "/tmp/consul/hcp-metrics/gqmuzdHCUPAEY5mbF8vgkZCNI14.sock" } } } @@ -654,7 +654,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { "endpoint": { "address": { "pipe": { - "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + "path": "/tmp/consul/hcp-metrics/gqmuzdHCUPAEY5mbF8vgkZCNI14.sock" } } } diff --git a/command/connect/envoy/testdata/hcp-metrics.golden b/command/connect/envoy/testdata/hcp-metrics.golden index 563d662e4436..0739ab531dac 100644 --- a/command/connect/envoy/testdata/hcp-metrics.golden +++ b/command/connect/envoy/testdata/hcp-metrics.golden @@ -67,7 +67,7 @@ "endpoint": { "address": { "pipe": { - "path": "/tmp/consul/hcp-metrics/default_test-proxy.sock" + "path": "/tmp/consul/hcp-metrics/k3bWnyJyKvjUYXrBdOX2nXzSSCQ.sock" } } } From 0f23def80c22afefc5cdccdb2d2a7f34a950c7da Mon Sep 17 00:00:00 2001 From: Freddy Date: Tue, 9 May 2023 12:28:34 -0600 Subject: [PATCH 07/13] Post a PR comment if the backport runner fails (#17197) --- .github/workflows/backport-assistant.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/backport-assistant.yml b/.github/workflows/backport-assistant.yml index 547ab28b7287..10acd19e7ea7 100644 --- a/.github/workflows/backport-assistant.yml +++ b/.github/workflows/backport-assistant.yml @@ -28,3 +28,16 @@ jobs: BACKPORT_LABEL_REGEXP: "backport/(?P\\d+\\.\\d+)" BACKPORT_TARGET_TEMPLATE: "release/{{.target}}.x" GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + handle-failure: + needs: + - backport + if: always() && needs.backport.result == 'failure' + runs-on: ubuntu-latest + steps: + - name: Comment on PR + run: | + github_message="Backport failed @${{ github.event.sender.login }}. Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + curl -s -H "Authorization: token ${{ secrets.PR_COMMENT_TOKEN }}" \ + -X POST \ + -d "{ \"body\": \"${github_message}\"}" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}/comments" \ No newline at end of file From 40eefaba180075473cb459f8af1aab0dfdddf92e Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Tue, 9 May 2023 13:57:40 -0500 Subject: [PATCH 08/13] Reaper controller for cascading deletes of owner resources (#17256) --- agent/consul/server.go | 2 + .../services/resource/write_status.go | 10 +- .../services/resource/write_status_test.go | 3 + internal/resource/reaper/controller.go | 148 +++++ internal/resource/reaper/controller_test.go | 193 ++++++ proto-public/pbresource/resource.pb.go | 626 +++++++++--------- proto-public/pbresource/resource.proto | 12 + proto-public/pbresource/resource_grpc.pb.go | 16 + 8 files changed, 705 insertions(+), 305 deletions(-) create mode 100644 internal/resource/reaper/controller.go create mode 100644 internal/resource/reaper/controller_test.go diff --git a/agent/consul/server.go b/agent/consul/server.go index 73df01e858e3..5ea0da7446d1 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -73,6 +73,7 @@ import ( "github.com/hashicorp/consul/internal/mesh" "github.com/hashicorp/consul/internal/resource" "github.com/hashicorp/consul/internal/resource/demo" + "github.com/hashicorp/consul/internal/resource/reaper" raftstorage "github.com/hashicorp/consul/internal/storage/raft" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/lib/routine" @@ -844,6 +845,7 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom func (s *Server) registerResources() { catalog.RegisterTypes(s.typeRegistry) mesh.RegisterTypes(s.typeRegistry) + reaper.RegisterControllers(s.controllerManager) if s.config.DevMode { demo.RegisterTypes(s.typeRegistry) diff --git a/agent/grpc-external/services/resource/write_status.go b/agent/grpc-external/services/resource/write_status.go index ec7c0da2f36e..205918e1dc2b 100644 --- a/agent/grpc-external/services/resource/write_status.go +++ b/agent/grpc-external/services/resource/write_status.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/oklog/ulid/v2" @@ -76,7 +77,10 @@ func (s *Server) WriteStatus(ctx context.Context, req *pbresource.WriteStatusReq if resource.Status == nil { resource.Status = make(map[string]*pbresource.Status) } - resource.Status[req.Key] = req.Status + + status := clone(req.Status) + status.UpdatedAt = timestamppb.Now() + resource.Status[req.Key] = status result, err = s.Backend.WriteCAS(ctx, resource) return err @@ -142,6 +146,10 @@ func validateWriteStatusRequest(req *pbresource.WriteStatusRequest) error { return status.Errorf(codes.InvalidArgument, "%s is required", field) } + if req.Status.UpdatedAt != nil { + return status.Error(codes.InvalidArgument, "status.updated_at is automatically set and cannot be provided") + } + if _, err := ulid.ParseStrict(req.Status.ObservedGeneration); err != nil { return status.Error(codes.InvalidArgument, "status.observed_generation is not valid") } diff --git a/agent/grpc-external/services/resource/write_status_test.go b/agent/grpc-external/services/resource/write_status_test.go index c5cc301c1ed9..f65c7918ff79 100644 --- a/agent/grpc-external/services/resource/write_status_test.go +++ b/agent/grpc-external/services/resource/write_status_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/hashicorp/consul/acl/resolver" "github.com/hashicorp/consul/internal/resource" @@ -85,6 +86,7 @@ func TestWriteStatus_InputValidation(t *testing.T) { "no reference type": func(req *pbresource.WriteStatusRequest) { req.Status.Conditions[0].Resource.Type = nil }, "no reference tenancy": func(req *pbresource.WriteStatusRequest) { req.Status.Conditions[0].Resource.Tenancy = nil }, "no reference name": func(req *pbresource.WriteStatusRequest) { req.Status.Conditions[0].Resource.Name = "" }, + "updated at provided": func(req *pbresource.WriteStatusRequest) { req.Status.UpdatedAt = timestamppb.Now() }, } for desc, modFn := range testCases { t.Run(desc, func(t *testing.T) { @@ -140,6 +142,7 @@ func TestWriteStatus_Success(t *testing.T) { require.NotEqual(t, rsp.Resource.Version, res.Version, "version should have changed") require.Contains(t, rsp.Resource.Status, "consul.io/other-controller") require.Contains(t, rsp.Resource.Status, "consul.io/artist-controller") + require.NotNil(t, rsp.Resource.Status["consul.io/artist-controller"].UpdatedAt) }) } } diff --git a/internal/resource/reaper/controller.go b/internal/resource/reaper/controller.go new file mode 100644 index 000000000000..7a7789f986a0 --- /dev/null +++ b/internal/resource/reaper/controller.go @@ -0,0 +1,148 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package reaper + +import ( + "context" + "time" + + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/resource" + "github.com/hashicorp/consul/proto-public/pbresource" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + statusKeyReaperController = "consul.io/reaper-controller" + secondPassDelay = 30 * time.Second + conditionTypeFirstPassCompleted = "FirstPassCompleted" +) + +// RegisterControllers registers controllers for the tombstone type. +func RegisterControllers(mgr *controller.Manager) { + mgr.Register(reaperController()) +} + +func reaperController() controller.Controller { + return controller.ForType(resource.TypeV1Tombstone). + WithReconciler(newReconciler()) +} + +func newReconciler() *tombstoneReconciler { + return &tombstoneReconciler{ + timeNow: time.Now, + } +} + +type tombstoneReconciler struct { + // Testing shim + timeNow func() time.Time +} + +// Deletes all owned (child) resources of an owner (parent) resource. +// +// The reconciliation for tombstones is split into two passes. +// The first pass attempts to delete child resources created before the owner resource was deleted. +// The second pass is run after a reasonable delay to delete child resources that may have been +// created during or after the completion of the first pass. +func (r *tombstoneReconciler) Reconcile(ctx context.Context, rt controller.Runtime, req controller.Request) error { + rsp, err := rt.Client.Read(ctx, &pbresource.ReadRequest{Id: req.ID}) + switch { + case status.Code(err) == codes.NotFound: + // tombstone not found. nothing to do + return nil + case err != nil: + // retry later + return err + } + res := rsp.Resource + + var tombstone pbresource.Tombstone + if err := res.Data.UnmarshalTo(&tombstone); err != nil { + return err + } + + firstPassCompletedOnEntry := isFirstPassCompleted(res) + + // Corner case: + // Check secondPassDelay has elasped since first pass in cases where queued + // reconciliation requests are lost between the first and second pass + // (e.g. controller relocated to a different server due to raft leadership + // change). + if firstPassCompletedOnEntry && !r.secondPassDelayElapsed(res.Status[statusKeyReaperController]) { + return controller.RequeueAfter(secondPassDelay) + } + + // Retrieve owner's children + listRsp, err := rt.Client.ListByOwner(ctx, &pbresource.ListByOwnerRequest{Owner: tombstone.Owner}) + if err != nil { + return err + } + + // Attempt to delete each child + for _, child := range listRsp.Resources { + _, err := rt.Client.Delete(ctx, &pbresource.DeleteRequest{Id: child.Id}) + if err != nil { + return err + } + } + + if firstPassCompletedOnEntry { + // we just did the second pass -> delete tombstone + _, err := rt.Client.Delete(ctx, &pbresource.DeleteRequest{Id: res.Id}) + if err != nil { + // tombstone deletion failed, just retry + return err + } + // tombstone delete succeeded and reconciliation complete + return nil + } else { + // we just did the first pass -> queue up the second pass + _, err = rt.Client.WriteStatus(ctx, &pbresource.WriteStatusRequest{ + Id: res.Id, + Key: statusKeyReaperController, + Status: &pbresource.Status{ + ObservedGeneration: res.Generation, + Conditions: []*pbresource.Condition{ + { + Type: conditionTypeFirstPassCompleted, + State: pbresource.Condition_STATE_TRUE, + Reason: "Success", + Message: "First pass of child resource deletion completed", + }, + }, + }, + }) + if err != nil { + return err + } + return controller.RequeueAfter(secondPassDelay) + } +} + +func (r *tombstoneReconciler) secondPassDelayElapsed(status *pbresource.Status) bool { + firstPassTime := status.UpdatedAt.AsTime() + return firstPassTime.Add(secondPassDelay).Before(r.timeNow()) +} + +func isFirstPassCompleted(res *pbresource.Resource) bool { + if res.Status == nil { + return false + } + + status, ok := res.Status[statusKeyReaperController] + if !ok { + return false + } + + // First time through, first and second pass ahead of us + if len(status.Conditions) == 0 { + return false + } + + // Single condition "FirstPassCompleted" + condition := status.Conditions[0] + return condition.State == pbresource.Condition_STATE_TRUE +} diff --git a/internal/resource/reaper/controller_test.go b/internal/resource/reaper/controller_test.go new file mode 100644 index 000000000000..9e6f0f3d5a07 --- /dev/null +++ b/internal/resource/reaper/controller_test.go @@ -0,0 +1,193 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package reaper + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing" + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/resource" + "github.com/hashicorp/consul/internal/resource/demo" + "github.com/hashicorp/consul/proto-public/pbresource" + "github.com/hashicorp/consul/sdk/testutil" +) + +func TestReconcile_ResourceWithNoChildren(t *testing.T) { + client := svctest.RunResourceService(t, demo.RegisterTypes) + + // Seed the database with an artist. + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + ctx := testutil.TestContext(t) + writeRsp, err := client.Write(ctx, &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + // Delete the artist to create a tombstone + _, err = client.Delete(ctx, &pbresource.DeleteRequest{Id: writeRsp.Resource.Id}) + require.NoError(t, err) + + // Retrieve tombstone + listRsp, err := client.List(ctx, &pbresource.ListRequest{ + Type: resource.TypeV1Tombstone, + Tenancy: writeRsp.Resource.Id.Tenancy, + }) + require.NoError(t, err) + require.Len(t, listRsp.Resources, 1) + tombstone := listRsp.Resources[0] + + // Verify reconcile does first pass and queues up for a second pass + rec := newReconciler() + runtime := controller.Runtime{ + Client: client, + Logger: testutil.Logger(t), + } + req := controller.Request{ID: tombstone.Id} + require.ErrorIs(t, controller.RequeueAfterError(secondPassDelay), rec.Reconcile(ctx, runtime, req)) + + // Verify condition FirstPassCompleted is true + readRsp, err := client.Read(ctx, &pbresource.ReadRequest{Id: tombstone.Id}) + require.NoError(t, err) + tombstone = readRsp.Resource + condition := tombstone.Status[statusKeyReaperController].Conditions[0] + require.Equal(t, conditionTypeFirstPassCompleted, condition.Type) + require.Equal(t, pbresource.Condition_STATE_TRUE, condition.State) + + // Verify reconcile does second pass and tombstone is deleted + // Fake out time so elapsed time > secondPassDelay + rec.timeNow = func() time.Time { return time.Now().Add(secondPassDelay + time.Second) } + require.NoError(t, rec.Reconcile(ctx, runtime, req)) + _, err = client.Read(ctx, &pbresource.ReadRequest{Id: tombstone.Id}) + require.Error(t, err) + require.Equal(t, codes.NotFound.String(), status.Code(err).String()) + + // Reconcile again to verify no-op on an already deleted tombstone + require.NoError(t, rec.Reconcile(ctx, runtime, req)) +} + +func TestReconcile_ResourceWithChildren(t *testing.T) { + client := svctest.RunResourceService(t, demo.RegisterTypes) + + // Seed the database with an artist + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + ctx := testutil.TestContext(t) + writeRsp, err := client.Write(ctx, &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + artist := writeRsp.Resource + + // Create 3 albums owned by the artist + numAlbums := 3 + for i := 0; i < numAlbums; i++ { + res, err = demo.GenerateV2Album(artist.Id) + require.NoError(t, err) + _, err := client.Write(ctx, &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + } + + // Delete the artist to create a tombstone + _, err = client.Delete(ctx, &pbresource.DeleteRequest{Id: writeRsp.Resource.Id}) + require.NoError(t, err) + + // Retrieve the tombstone + listRsp, err := client.List(ctx, &pbresource.ListRequest{ + Type: resource.TypeV1Tombstone, + Tenancy: writeRsp.Resource.Id.Tenancy, + }) + require.NoError(t, err) + require.Len(t, listRsp.Resources, 1) + tombstone := listRsp.Resources[0] + + // Verify reconcile does first pass delete and queues up for a second pass + rec := newReconciler() + runtime := controller.Runtime{ + Client: client, + Logger: testutil.Logger(t), + } + req := controller.Request{ID: tombstone.Id} + require.ErrorIs(t, controller.RequeueAfterError(secondPassDelay), rec.Reconcile(ctx, runtime, req)) + + // Verify 3 albums deleted + listRsp, err = client.List(ctx, &pbresource.ListRequest{ + Type: demo.TypeV2Album, + Tenancy: artist.Id.Tenancy, + }) + require.NoError(t, err) + require.Empty(t, listRsp.Resources) + + // Verify condition FirstPassCompleted is true + readRsp, err := client.Read(ctx, &pbresource.ReadRequest{Id: tombstone.Id}) + require.NoError(t, err) + tombstone = readRsp.Resource + condition := tombstone.Status[statusKeyReaperController].Conditions[0] + require.Equal(t, conditionTypeFirstPassCompleted, condition.Type) + require.Equal(t, pbresource.Condition_STATE_TRUE, condition.State) + + // Verify reconcile does second pass + // Fake out time so elapsed time > secondPassDelay + rec.timeNow = func() time.Time { return time.Now().Add(secondPassDelay + time.Second) } + require.NoError(t, rec.Reconcile(ctx, runtime, req)) + + // Verify artist tombstone deleted + _, err = client.Read(ctx, &pbresource.ReadRequest{Id: tombstone.Id}) + require.Error(t, err) + require.Equal(t, codes.NotFound.String(), status.Code(err).String()) + + // Verify tombstones for 3 albums created + listRsp, err = client.List(ctx, &pbresource.ListRequest{ + Type: resource.TypeV1Tombstone, + Tenancy: artist.Id.Tenancy, + }) + require.NoError(t, err) + require.Len(t, listRsp.Resources, numAlbums) +} + +func TestReconcile_RequeueWithDelayWhenSecondPassDelayNotElapsed(t *testing.T) { + client := svctest.RunResourceService(t, demo.RegisterTypes) + + // Seed the database with an artist. + res, err := demo.GenerateV2Artist() + require.NoError(t, err) + ctx := testutil.TestContext(t) + writeRsp, err := client.Write(ctx, &pbresource.WriteRequest{Resource: res}) + require.NoError(t, err) + + // Delete the artist to create a tombstone + _, err = client.Delete(ctx, &pbresource.DeleteRequest{Id: writeRsp.Resource.Id}) + require.NoError(t, err) + + // Retrieve tombstone + listRsp, err := client.List(ctx, &pbresource.ListRequest{ + Type: resource.TypeV1Tombstone, + Tenancy: writeRsp.Resource.Id.Tenancy, + }) + require.NoError(t, err) + require.Len(t, listRsp.Resources, 1) + tombstone := listRsp.Resources[0] + + // Verify reconcile does first pass and queues up for a second pass + rec := newReconciler() + runtime := controller.Runtime{ + Client: client, + Logger: testutil.Logger(t), + } + req := controller.Request{ID: tombstone.Id} + require.ErrorIs(t, controller.RequeueAfterError(secondPassDelay), rec.Reconcile(ctx, runtime, req)) + + // Verify condition FirstPassCompleted is true + readRsp, err := client.Read(ctx, &pbresource.ReadRequest{Id: tombstone.Id}) + require.NoError(t, err) + tombstone = readRsp.Resource + condition := tombstone.Status[statusKeyReaperController].Conditions[0] + require.Equal(t, conditionTypeFirstPassCompleted, condition.Type) + require.Equal(t, pbresource.Condition_STATE_TRUE, condition.State) + + // Verify requeued for second pass since secondPassDelay time has not elapsed + require.ErrorIs(t, controller.RequeueAfterError(secondPassDelay), rec.Reconcile(ctx, runtime, req)) +} diff --git a/proto-public/pbresource/resource.pb.go b/proto-public/pbresource/resource.pb.go index d05a2b8f972b..76bf1be45c56 100644 --- a/proto-public/pbresource/resource.pb.go +++ b/proto-public/pbresource/resource.pb.go @@ -14,6 +14,7 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" anypb "google.golang.org/protobuf/types/known/anypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -505,6 +506,8 @@ type Status struct { // Conditions contains a set of discreet observations about the resource in // relation to the current state of the system (e.g. it is semantically valid). Conditions []*Condition `protobuf:"bytes,2,rep,name=conditions,proto3" json:"conditions,omitempty"` + // UpdatedAt is the time at which the status was last written. + UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` } func (x *Status) Reset() { @@ -553,6 +556,13 @@ func (x *Status) GetConditions() []*Condition { return nil } +func (x *Status) GetUpdatedAt() *timestamppb.Timestamp { + if x != nil { + return x.UpdatedAt + } + return nil +} + // Condition represents a discreet observation about a resource in relation to // the current state of the system. // @@ -1552,259 +1562,265 @@ var file_pbresource_resource_proto_rawDesc = []byte{ 0x6f, 0x6e, 0x73, 0x2f, 0x72, 0x61, 0x74, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x2f, 0x72, 0x61, 0x74, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, - 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x55, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x23, 0x0a, 0x0d, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x67, - 0x72, 0x6f, 0x75, 0x70, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6b, - 0x69, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, - 0x62, 0x0a, 0x07, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x22, 0x9d, 0x01, 0x0a, 0x02, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, - 0x6e, 0x63, 0x79, 0x22, 0x85, 0x04, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x12, 0x2d, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x33, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x05, 0x6f, - 0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, - 0x0a, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4d, - 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x47, 0x0a, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5c, 0x0a, - 0x0b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x37, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7f, 0x0a, 0x06, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x64, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x12, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x47, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x92, 0x02, 0x0a, - 0x09, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x40, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x40, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x22, 0x3b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x11, 0x0a, - 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, - 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x52, 0x55, 0x45, 0x10, 0x01, - 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x53, 0x45, 0x10, - 0x02, 0x22, 0xac, 0x01, 0x0a, 0x09, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, - 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, - 0x63, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x22, 0x40, 0x0a, 0x09, 0x54, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, 0x65, 0x12, 0x33, 0x0a, - 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x05, 0x6f, 0x77, 0x6e, - 0x65, 0x72, 0x22, 0x3c, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x2d, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, - 0x22, 0x4f, 0x0a, 0x0c, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x3f, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x22, 0xa1, 0x01, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x55, 0x0a, 0x04, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x23, 0x0a, 0x0d, 0x67, 0x72, 0x6f, 0x75, 0x70, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, + 0x22, 0x62, 0x0a, 0x07, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x65, 0x65, 0x72, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, + 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x9d, 0x01, 0x0a, 0x02, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x75, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, - 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x52, 0x07, 0x74, 0x65, 0x6e, - 0x61, 0x6e, 0x63, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x70, 0x72, 0x65, - 0x66, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x50, - 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x51, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x49, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, - 0x42, 0x79, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, - 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x05, 0x6f, 0x77, - 0x6e, 0x65, 0x72, 0x22, 0x58, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x4f, 0x77, 0x6e, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x09, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x61, 0x6e, 0x63, 0x79, 0x22, 0x85, 0x04, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x12, 0x2d, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x4f, 0x0a, - 0x0c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, + 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x33, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x05, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1e, 0x0a, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x4d, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x47, + 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5c, + 0x0a, 0x0b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xba, 0x01, 0x0a, + 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x6f, 0x62, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x64, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x39, + 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x92, 0x02, 0x0a, 0x09, 0x43, 0x6f, + 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x40, 0x0a, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x40, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x24, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x22, 0x3b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0e, 0x0a, + 0x0a, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x52, 0x55, 0x45, 0x10, 0x01, 0x12, 0x0f, 0x0a, + 0x0b, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x53, 0x45, 0x10, 0x02, 0x22, 0xac, + 0x01, 0x0a, 0x09, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x33, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, + 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x40, 0x0a, + 0x09, 0x54, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, 0x65, 0x12, 0x33, 0x0a, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, + 0x3c, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x22, 0x4f, 0x0a, + 0x0c, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x50, - 0x0a, 0x0d, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3f, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x22, 0xaa, 0x01, 0x0a, 0x12, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, - 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x39, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x56, 0x0a, - 0x13, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x58, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, - 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, - 0x10, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0xa6, 0x01, 0x0a, 0x10, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x74, - 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, - 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x61, 0x6d, - 0x65, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x6e, 0x61, 0x6d, 0x65, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0xf0, 0x01, 0x0a, 0x0a, 0x57, - 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x4d, 0x0a, 0x09, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2f, 0x2e, 0x68, + 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xa1, + 0x01, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, + 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x50, 0x72, 0x65, 0x66, + 0x69, 0x78, 0x22, 0x51, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x41, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x49, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x4f, + 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x22, 0x58, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x52, 0x0a, 0x09, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x19, 0x0a, 0x15, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, - 0x00, 0x12, 0x14, 0x0a, 0x10, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, - 0x50, 0x53, 0x45, 0x52, 0x54, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x4f, 0x50, 0x45, 0x52, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x32, 0x83, 0x06, - 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x61, 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x26, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x4f, 0x0a, 0x0c, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x08, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x50, 0x0a, 0x0d, 0x57, + 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x08, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xaa, 0x01, + 0x0a, 0x12, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x39, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x21, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x56, 0x0a, 0x13, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x3f, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x22, 0x58, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x49, 0x44, 0x52, 0x02, + 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x10, 0x0a, 0x0e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xa6, + 0x01, 0x0a, 0x10, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, + 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x27, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, - 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, - 0x08, 0x02, 0x10, 0x0b, 0x12, 0x64, 0x0a, 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x27, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x03, 0x10, 0x0b, 0x12, 0x76, 0x0a, 0x0b, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x52, 0x07, 0x74, + 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x70, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, + 0x65, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0xf0, 0x01, 0x0a, 0x0a, 0x57, 0x61, 0x74, 0x63, + 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x4d, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x03, - 0x10, 0x0b, 0x12, 0x61, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x26, 0x2e, 0x68, 0x61, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x52, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x19, 0x0a, 0x15, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, + 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, + 0x0a, 0x10, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x50, 0x53, 0x45, + 0x52, 0x54, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, + 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x32, 0x83, 0x06, 0x0a, 0x0f, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x61, + 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x26, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x02, 0x10, + 0x0b, 0x12, 0x64, 0x0a, 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x27, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, - 0x04, 0x08, 0x02, 0x10, 0x0b, 0x12, 0x76, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x4f, - 0x77, 0x6e, 0x65, 0x72, 0x12, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x02, 0x10, 0x0b, 0x12, 0x67, 0x0a, - 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x28, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, + 0x86, 0x04, 0x04, 0x08, 0x03, 0x10, 0x0b, 0x12, 0x76, 0x0a, 0x0b, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x03, 0x10, 0x0b, 0x12, + 0x61, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x26, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x29, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, - 0x04, 0x04, 0x08, 0x03, 0x10, 0x0b, 0x12, 0x6b, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, - 0x69, 0x73, 0x74, 0x12, 0x2b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, - 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x27, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x02, + 0x10, 0x0b, 0x12, 0x76, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x4f, 0x77, 0x6e, 0x65, + 0x72, 0x12, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x42, 0x79, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x2e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x42, 0x79, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x02, 0x10, 0x0b, 0x12, 0x67, 0x0a, 0x06, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x12, 0x28, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, + 0x03, 0x10, 0x0b, 0x12, 0x6b, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x2b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x61, 0x74, - 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x02, 0x10, - 0x0b, 0x30, 0x01, 0x42, 0xe9, 0x01, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x0d, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x2f, 0x70, 0x62, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xa2, 0x02, 0x03, 0x48, 0x43, - 0x52, 0xaa, 0x02, 0x19, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xca, 0x02, 0x19, - 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x5c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xe2, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0xea, 0x02, 0x1b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x63, 0x68, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x02, 0x10, 0x0b, 0x30, 0x01, + 0x42, 0xe9, 0x01, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x42, 0x0d, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x01, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x2f, 0x70, 0x62, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xa2, 0x02, 0x03, 0x48, 0x43, 0x52, 0xaa, 0x02, + 0x19, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xca, 0x02, 0x19, 0x48, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xe2, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, + 0x1b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x3a, 0x3a, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1822,33 +1838,34 @@ func file_pbresource_resource_proto_rawDescGZIP() []byte { var file_pbresource_resource_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_pbresource_resource_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_pbresource_resource_proto_goTypes = []interface{}{ - (Condition_State)(0), // 0: hashicorp.consul.resource.Condition.State - (WatchEvent_Operation)(0), // 1: hashicorp.consul.resource.WatchEvent.Operation - (*Type)(nil), // 2: hashicorp.consul.resource.Type - (*Tenancy)(nil), // 3: hashicorp.consul.resource.Tenancy - (*ID)(nil), // 4: hashicorp.consul.resource.ID - (*Resource)(nil), // 5: hashicorp.consul.resource.Resource - (*Status)(nil), // 6: hashicorp.consul.resource.Status - (*Condition)(nil), // 7: hashicorp.consul.resource.Condition - (*Reference)(nil), // 8: hashicorp.consul.resource.Reference - (*Tombstone)(nil), // 9: hashicorp.consul.resource.Tombstone - (*ReadRequest)(nil), // 10: hashicorp.consul.resource.ReadRequest - (*ReadResponse)(nil), // 11: hashicorp.consul.resource.ReadResponse - (*ListRequest)(nil), // 12: hashicorp.consul.resource.ListRequest - (*ListResponse)(nil), // 13: hashicorp.consul.resource.ListResponse - (*ListByOwnerRequest)(nil), // 14: hashicorp.consul.resource.ListByOwnerRequest - (*ListByOwnerResponse)(nil), // 15: hashicorp.consul.resource.ListByOwnerResponse - (*WriteRequest)(nil), // 16: hashicorp.consul.resource.WriteRequest - (*WriteResponse)(nil), // 17: hashicorp.consul.resource.WriteResponse - (*WriteStatusRequest)(nil), // 18: hashicorp.consul.resource.WriteStatusRequest - (*WriteStatusResponse)(nil), // 19: hashicorp.consul.resource.WriteStatusResponse - (*DeleteRequest)(nil), // 20: hashicorp.consul.resource.DeleteRequest - (*DeleteResponse)(nil), // 21: hashicorp.consul.resource.DeleteResponse - (*WatchListRequest)(nil), // 22: hashicorp.consul.resource.WatchListRequest - (*WatchEvent)(nil), // 23: hashicorp.consul.resource.WatchEvent - nil, // 24: hashicorp.consul.resource.Resource.MetadataEntry - nil, // 25: hashicorp.consul.resource.Resource.StatusEntry - (*anypb.Any)(nil), // 26: google.protobuf.Any + (Condition_State)(0), // 0: hashicorp.consul.resource.Condition.State + (WatchEvent_Operation)(0), // 1: hashicorp.consul.resource.WatchEvent.Operation + (*Type)(nil), // 2: hashicorp.consul.resource.Type + (*Tenancy)(nil), // 3: hashicorp.consul.resource.Tenancy + (*ID)(nil), // 4: hashicorp.consul.resource.ID + (*Resource)(nil), // 5: hashicorp.consul.resource.Resource + (*Status)(nil), // 6: hashicorp.consul.resource.Status + (*Condition)(nil), // 7: hashicorp.consul.resource.Condition + (*Reference)(nil), // 8: hashicorp.consul.resource.Reference + (*Tombstone)(nil), // 9: hashicorp.consul.resource.Tombstone + (*ReadRequest)(nil), // 10: hashicorp.consul.resource.ReadRequest + (*ReadResponse)(nil), // 11: hashicorp.consul.resource.ReadResponse + (*ListRequest)(nil), // 12: hashicorp.consul.resource.ListRequest + (*ListResponse)(nil), // 13: hashicorp.consul.resource.ListResponse + (*ListByOwnerRequest)(nil), // 14: hashicorp.consul.resource.ListByOwnerRequest + (*ListByOwnerResponse)(nil), // 15: hashicorp.consul.resource.ListByOwnerResponse + (*WriteRequest)(nil), // 16: hashicorp.consul.resource.WriteRequest + (*WriteResponse)(nil), // 17: hashicorp.consul.resource.WriteResponse + (*WriteStatusRequest)(nil), // 18: hashicorp.consul.resource.WriteStatusRequest + (*WriteStatusResponse)(nil), // 19: hashicorp.consul.resource.WriteStatusResponse + (*DeleteRequest)(nil), // 20: hashicorp.consul.resource.DeleteRequest + (*DeleteResponse)(nil), // 21: hashicorp.consul.resource.DeleteResponse + (*WatchListRequest)(nil), // 22: hashicorp.consul.resource.WatchListRequest + (*WatchEvent)(nil), // 23: hashicorp.consul.resource.WatchEvent + nil, // 24: hashicorp.consul.resource.Resource.MetadataEntry + nil, // 25: hashicorp.consul.resource.Resource.StatusEntry + (*anypb.Any)(nil), // 26: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 27: google.protobuf.Timestamp } var file_pbresource_resource_proto_depIdxs = []int32{ 2, // 0: hashicorp.consul.resource.ID.type:type_name -> hashicorp.consul.resource.Type @@ -1859,48 +1876,49 @@ var file_pbresource_resource_proto_depIdxs = []int32{ 25, // 5: hashicorp.consul.resource.Resource.status:type_name -> hashicorp.consul.resource.Resource.StatusEntry 26, // 6: hashicorp.consul.resource.Resource.data:type_name -> google.protobuf.Any 7, // 7: hashicorp.consul.resource.Status.conditions:type_name -> hashicorp.consul.resource.Condition - 0, // 8: hashicorp.consul.resource.Condition.state:type_name -> hashicorp.consul.resource.Condition.State - 8, // 9: hashicorp.consul.resource.Condition.resource:type_name -> hashicorp.consul.resource.Reference - 2, // 10: hashicorp.consul.resource.Reference.type:type_name -> hashicorp.consul.resource.Type - 3, // 11: hashicorp.consul.resource.Reference.tenancy:type_name -> hashicorp.consul.resource.Tenancy - 4, // 12: hashicorp.consul.resource.Tombstone.owner:type_name -> hashicorp.consul.resource.ID - 4, // 13: hashicorp.consul.resource.ReadRequest.id:type_name -> hashicorp.consul.resource.ID - 5, // 14: hashicorp.consul.resource.ReadResponse.resource:type_name -> hashicorp.consul.resource.Resource - 2, // 15: hashicorp.consul.resource.ListRequest.type:type_name -> hashicorp.consul.resource.Type - 3, // 16: hashicorp.consul.resource.ListRequest.tenancy:type_name -> hashicorp.consul.resource.Tenancy - 5, // 17: hashicorp.consul.resource.ListResponse.resources:type_name -> hashicorp.consul.resource.Resource - 4, // 18: hashicorp.consul.resource.ListByOwnerRequest.owner:type_name -> hashicorp.consul.resource.ID - 5, // 19: hashicorp.consul.resource.ListByOwnerResponse.resources:type_name -> hashicorp.consul.resource.Resource - 5, // 20: hashicorp.consul.resource.WriteRequest.resource:type_name -> hashicorp.consul.resource.Resource - 5, // 21: hashicorp.consul.resource.WriteResponse.resource:type_name -> hashicorp.consul.resource.Resource - 4, // 22: hashicorp.consul.resource.WriteStatusRequest.id:type_name -> hashicorp.consul.resource.ID - 6, // 23: hashicorp.consul.resource.WriteStatusRequest.status:type_name -> hashicorp.consul.resource.Status - 5, // 24: hashicorp.consul.resource.WriteStatusResponse.resource:type_name -> hashicorp.consul.resource.Resource - 4, // 25: hashicorp.consul.resource.DeleteRequest.id:type_name -> hashicorp.consul.resource.ID - 2, // 26: hashicorp.consul.resource.WatchListRequest.type:type_name -> hashicorp.consul.resource.Type - 3, // 27: hashicorp.consul.resource.WatchListRequest.tenancy:type_name -> hashicorp.consul.resource.Tenancy - 1, // 28: hashicorp.consul.resource.WatchEvent.operation:type_name -> hashicorp.consul.resource.WatchEvent.Operation - 5, // 29: hashicorp.consul.resource.WatchEvent.resource:type_name -> hashicorp.consul.resource.Resource - 6, // 30: hashicorp.consul.resource.Resource.StatusEntry.value:type_name -> hashicorp.consul.resource.Status - 10, // 31: hashicorp.consul.resource.ResourceService.Read:input_type -> hashicorp.consul.resource.ReadRequest - 16, // 32: hashicorp.consul.resource.ResourceService.Write:input_type -> hashicorp.consul.resource.WriteRequest - 18, // 33: hashicorp.consul.resource.ResourceService.WriteStatus:input_type -> hashicorp.consul.resource.WriteStatusRequest - 12, // 34: hashicorp.consul.resource.ResourceService.List:input_type -> hashicorp.consul.resource.ListRequest - 14, // 35: hashicorp.consul.resource.ResourceService.ListByOwner:input_type -> hashicorp.consul.resource.ListByOwnerRequest - 20, // 36: hashicorp.consul.resource.ResourceService.Delete:input_type -> hashicorp.consul.resource.DeleteRequest - 22, // 37: hashicorp.consul.resource.ResourceService.WatchList:input_type -> hashicorp.consul.resource.WatchListRequest - 11, // 38: hashicorp.consul.resource.ResourceService.Read:output_type -> hashicorp.consul.resource.ReadResponse - 17, // 39: hashicorp.consul.resource.ResourceService.Write:output_type -> hashicorp.consul.resource.WriteResponse - 19, // 40: hashicorp.consul.resource.ResourceService.WriteStatus:output_type -> hashicorp.consul.resource.WriteStatusResponse - 13, // 41: hashicorp.consul.resource.ResourceService.List:output_type -> hashicorp.consul.resource.ListResponse - 15, // 42: hashicorp.consul.resource.ResourceService.ListByOwner:output_type -> hashicorp.consul.resource.ListByOwnerResponse - 21, // 43: hashicorp.consul.resource.ResourceService.Delete:output_type -> hashicorp.consul.resource.DeleteResponse - 23, // 44: hashicorp.consul.resource.ResourceService.WatchList:output_type -> hashicorp.consul.resource.WatchEvent - 38, // [38:45] is the sub-list for method output_type - 31, // [31:38] is the sub-list for method input_type - 31, // [31:31] is the sub-list for extension type_name - 31, // [31:31] is the sub-list for extension extendee - 0, // [0:31] is the sub-list for field type_name + 27, // 8: hashicorp.consul.resource.Status.updated_at:type_name -> google.protobuf.Timestamp + 0, // 9: hashicorp.consul.resource.Condition.state:type_name -> hashicorp.consul.resource.Condition.State + 8, // 10: hashicorp.consul.resource.Condition.resource:type_name -> hashicorp.consul.resource.Reference + 2, // 11: hashicorp.consul.resource.Reference.type:type_name -> hashicorp.consul.resource.Type + 3, // 12: hashicorp.consul.resource.Reference.tenancy:type_name -> hashicorp.consul.resource.Tenancy + 4, // 13: hashicorp.consul.resource.Tombstone.owner:type_name -> hashicorp.consul.resource.ID + 4, // 14: hashicorp.consul.resource.ReadRequest.id:type_name -> hashicorp.consul.resource.ID + 5, // 15: hashicorp.consul.resource.ReadResponse.resource:type_name -> hashicorp.consul.resource.Resource + 2, // 16: hashicorp.consul.resource.ListRequest.type:type_name -> hashicorp.consul.resource.Type + 3, // 17: hashicorp.consul.resource.ListRequest.tenancy:type_name -> hashicorp.consul.resource.Tenancy + 5, // 18: hashicorp.consul.resource.ListResponse.resources:type_name -> hashicorp.consul.resource.Resource + 4, // 19: hashicorp.consul.resource.ListByOwnerRequest.owner:type_name -> hashicorp.consul.resource.ID + 5, // 20: hashicorp.consul.resource.ListByOwnerResponse.resources:type_name -> hashicorp.consul.resource.Resource + 5, // 21: hashicorp.consul.resource.WriteRequest.resource:type_name -> hashicorp.consul.resource.Resource + 5, // 22: hashicorp.consul.resource.WriteResponse.resource:type_name -> hashicorp.consul.resource.Resource + 4, // 23: hashicorp.consul.resource.WriteStatusRequest.id:type_name -> hashicorp.consul.resource.ID + 6, // 24: hashicorp.consul.resource.WriteStatusRequest.status:type_name -> hashicorp.consul.resource.Status + 5, // 25: hashicorp.consul.resource.WriteStatusResponse.resource:type_name -> hashicorp.consul.resource.Resource + 4, // 26: hashicorp.consul.resource.DeleteRequest.id:type_name -> hashicorp.consul.resource.ID + 2, // 27: hashicorp.consul.resource.WatchListRequest.type:type_name -> hashicorp.consul.resource.Type + 3, // 28: hashicorp.consul.resource.WatchListRequest.tenancy:type_name -> hashicorp.consul.resource.Tenancy + 1, // 29: hashicorp.consul.resource.WatchEvent.operation:type_name -> hashicorp.consul.resource.WatchEvent.Operation + 5, // 30: hashicorp.consul.resource.WatchEvent.resource:type_name -> hashicorp.consul.resource.Resource + 6, // 31: hashicorp.consul.resource.Resource.StatusEntry.value:type_name -> hashicorp.consul.resource.Status + 10, // 32: hashicorp.consul.resource.ResourceService.Read:input_type -> hashicorp.consul.resource.ReadRequest + 16, // 33: hashicorp.consul.resource.ResourceService.Write:input_type -> hashicorp.consul.resource.WriteRequest + 18, // 34: hashicorp.consul.resource.ResourceService.WriteStatus:input_type -> hashicorp.consul.resource.WriteStatusRequest + 12, // 35: hashicorp.consul.resource.ResourceService.List:input_type -> hashicorp.consul.resource.ListRequest + 14, // 36: hashicorp.consul.resource.ResourceService.ListByOwner:input_type -> hashicorp.consul.resource.ListByOwnerRequest + 20, // 37: hashicorp.consul.resource.ResourceService.Delete:input_type -> hashicorp.consul.resource.DeleteRequest + 22, // 38: hashicorp.consul.resource.ResourceService.WatchList:input_type -> hashicorp.consul.resource.WatchListRequest + 11, // 39: hashicorp.consul.resource.ResourceService.Read:output_type -> hashicorp.consul.resource.ReadResponse + 17, // 40: hashicorp.consul.resource.ResourceService.Write:output_type -> hashicorp.consul.resource.WriteResponse + 19, // 41: hashicorp.consul.resource.ResourceService.WriteStatus:output_type -> hashicorp.consul.resource.WriteStatusResponse + 13, // 42: hashicorp.consul.resource.ResourceService.List:output_type -> hashicorp.consul.resource.ListResponse + 15, // 43: hashicorp.consul.resource.ResourceService.ListByOwner:output_type -> hashicorp.consul.resource.ListByOwnerResponse + 21, // 44: hashicorp.consul.resource.ResourceService.Delete:output_type -> hashicorp.consul.resource.DeleteResponse + 23, // 45: hashicorp.consul.resource.ResourceService.WatchList:output_type -> hashicorp.consul.resource.WatchEvent + 39, // [39:46] is the sub-list for method output_type + 32, // [32:39] is the sub-list for method input_type + 32, // [32:32] is the sub-list for extension type_name + 32, // [32:32] is the sub-list for extension extendee + 0, // [0:32] is the sub-list for field type_name } func init() { file_pbresource_resource_proto_init() } diff --git a/proto-public/pbresource/resource.proto b/proto-public/pbresource/resource.proto index b3f7a039cd75..038f96c7a5bd 100644 --- a/proto-public/pbresource/resource.proto +++ b/proto-public/pbresource/resource.proto @@ -7,6 +7,7 @@ package hashicorp.consul.resource; import "annotations/ratelimit/ratelimit.proto"; import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; // Type describes a resource's type. It follows the GVK (Group Version Kind) // [pattern](https://book.kubebuilder.io/cronjob-tutorial/gvks.html) established @@ -124,6 +125,9 @@ message Status { // Conditions contains a set of discreet observations about the resource in // relation to the current state of the system (e.g. it is semantically valid). repeated Condition conditions = 2; + + // UpdatedAt is the time at which the status was last written. + google.protobuf.Timestamp updated_at = 3; } // Condition represents a discreet observation about a resource in relation to @@ -224,6 +228,14 @@ service ResourceService { // By default, reads are eventually consistent, but you can opt-in to strong // consistency via the x-consul-consistency-mode metadata (see ResourceService // docs for more info). + // + // Errors with NotFound if the resource is not found. + // + // Errors with InvalidArgument if the request fails validation or the resource + // is stored as a type with a different GroupVersion than was requested. + // + // Errors with PermissionDenied if the caller is not authorized to read + // the resource. rpc Read(ReadRequest) returns (ReadResponse) { option (hashicorp.consul.internal.ratelimit.spec) = { operation_type: OPERATION_TYPE_READ, diff --git a/proto-public/pbresource/resource_grpc.pb.go b/proto-public/pbresource/resource_grpc.pb.go index af51fba75bdd..d15b677a270e 100644 --- a/proto-public/pbresource/resource_grpc.pb.go +++ b/proto-public/pbresource/resource_grpc.pb.go @@ -27,6 +27,14 @@ type ResourceServiceClient interface { // By default, reads are eventually consistent, but you can opt-in to strong // consistency via the x-consul-consistency-mode metadata (see ResourceService // docs for more info). + // + // Errors with NotFound if the resource is not found. + // + // Errors with InvalidArgument if the request fails validation or the resource + // is stored as a type with a different GroupVersion than was requested. + // + // Errors with PermissionDenied if the caller is not authorized to read + // the resource. Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) // Write a resource. // @@ -204,6 +212,14 @@ type ResourceServiceServer interface { // By default, reads are eventually consistent, but you can opt-in to strong // consistency via the x-consul-consistency-mode metadata (see ResourceService // docs for more info). + // + // Errors with NotFound if the resource is not found. + // + // Errors with InvalidArgument if the request fails validation or the resource + // is stored as a type with a different GroupVersion than was requested. + // + // Errors with PermissionDenied if the caller is not authorized to read + // the resource. Read(context.Context, *ReadRequest) (*ReadResponse, error) // Write a resource. // From 48f7d99305d9c13d40fec88cb40bde496996699c Mon Sep 17 00:00:00 2001 From: cskh Date: Tue, 9 May 2023 15:28:52 -0400 Subject: [PATCH 09/13] snapshot: some improvments to the snapshot process (#17236) * snapshot: some improvments to the snapshot process Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Chris S. Kim --- .changelog/17236.txt | 3 +++ agent/consul/server.go | 2 +- snapshot/snapshot.go | 3 ++- website/content/commands/snapshot/save.mdx | 4 +++- 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .changelog/17236.txt diff --git a/.changelog/17236.txt b/.changelog/17236.txt new file mode 100644 index 000000000000..c824bb7ed782 --- /dev/null +++ b/.changelog/17236.txt @@ -0,0 +1,3 @@ +```release-note:improvement +logging: change snapshot log header from `agent.server.snapshot` to `agent.server.raft.snapshot` +``` diff --git a/agent/consul/server.go b/agent/consul/server.go index 5ea0da7446d1..52236e8b5d00 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -1056,7 +1056,7 @@ func (s *Server) setupRaft() error { log = cacheStore // Create the snapshot store. - snapshots, err := raft.NewFileSnapshotStoreWithLogger(path, snapshotsRetained, s.logger.Named("snapshot")) + snapshots, err := raft.NewFileSnapshotStoreWithLogger(path, snapshotsRetained, s.logger.Named("raft.snapshot")) if err != nil { return err } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index fbe8660f0f2a..a1deee9153dc 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -53,6 +53,7 @@ func New(logger hclog.Logger, r *raft.Raft) (*Snapshot, error) { if err != nil { return nil, fmt.Errorf("failed to create snapshot file: %v", err) } + logger.Debug("creating temporary file of snapshot", "path", archive.Name()) // If anything goes wrong after this point, we will attempt to clean up // the temp file. The happy path will disarm this. @@ -112,7 +113,7 @@ func (s *Snapshot) Read(p []byte) (n int, err error) { } // Close closes the snapshot and removes any temporary storage associated with -// it. You must arrange to call this whenever NewSnapshot() has been called +// it. You must arrange to call this whenever New() has been called // successfully. This is safe to call on a nil snapshot. func (s *Snapshot) Close() error { if s == nil { diff --git a/website/content/commands/snapshot/save.mdx b/website/content/commands/snapshot/save.mdx index 63b0fbe48f3d..18ffc50150ab 100644 --- a/website/content/commands/snapshot/save.mdx +++ b/website/content/commands/snapshot/save.mdx @@ -20,7 +20,8 @@ If ACLs are enabled, a management token must be supplied in order to perform a snapshot save. -> Note that saving a snapshot involves the server process writing the snapshot to a -temporary file on-disk before sending that file to the CLI client. The default location +temporary file on-disk before sending that file to the CLI client. Upon successful completion, +Consul removes the temporary file. The default location of the temporary file can vary depending on operating system, but typically is `/tmp`. You can get more detailed information on default locations in the Go documentation for [os.TempDir](https://golang.org/pkg/os/#TempDir). If you need to change this location, you can do so by setting the `TMPDIR` environment @@ -28,6 +29,7 @@ variable for the Consul server processes. Keep in mind that setting the environm the CLI client attempting to perform a snapshot save will have no effect. It _must_ be set in the context of the server process. If you're using Systemd to manage your Consul server processes, then adding `Environment=TMPDIR=/path/to/dir` to your Consul unit file will work. +As a result of the Raft snapshot, Consul also saves one snapshot file at `data_dir/raft/snapshots`. The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) From 5ecab506a6748670ddb95405924eac9cdc50f02b Mon Sep 17 00:00:00 2001 From: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Date: Tue, 9 May 2023 16:36:29 -0500 Subject: [PATCH 10/13] Fix ent bug caused by #17241. (#17278) Fix ent bug caused by #17241 All tests passed in OSS, but not ENT. This is a patch to resolve the problem for both. --- agent/consul/health_endpoint.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/agent/consul/health_endpoint.go b/agent/consul/health_endpoint.go index 6913136d3844..945b3b2eb82a 100644 --- a/agent/consul/health_endpoint.go +++ b/agent/consul/health_endpoint.go @@ -214,6 +214,14 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc f = h.serviceNodesDefault } + authzContext := acl.AuthorizerContext{ + Peer: args.PeerName, + } + authz, err := h.srv.ResolveTokenAndDefaultMeta(args.Token, &args.EnterpriseMeta, &authzContext) + if err != nil { + return err + } + if err := h.srv.validateEnterpriseRequest(&args.EnterpriseMeta, false); err != nil { return err } @@ -239,14 +247,6 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc return err } - authzContext := acl.AuthorizerContext{ - Peer: args.PeerName, - } - authz, err := h.srv.ResolveTokenAndDefaultMeta(args.Token, &args.EnterpriseMeta, &authzContext) - if err != nil { - return err - } - // If we're doing a connect or ingress query, we need read access to the service // we're trying to find proxies for, so check that. if args.Connect || args.Ingress { From 6c24a66f73db1d261f4d25c49438b2fa2338f4aa Mon Sep 17 00:00:00 2001 From: Dan Upton Date: Wed, 10 May 2023 10:37:54 +0100 Subject: [PATCH 11/13] resource: optionally compare timestamps in `EqualStatus` (#17275) --- internal/resource/demo/controller.go | 2 +- internal/resource/equality.go | 13 +++++++++--- internal/resource/equality_test.go | 30 +++++++++++++++++++++------- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/internal/resource/demo/controller.go b/internal/resource/demo/controller.go index db935ec065ba..11afc3bac5e6 100644 --- a/internal/resource/demo/controller.go +++ b/internal/resource/demo/controller.go @@ -111,7 +111,7 @@ func (r *artistReconciler) Reconcile(ctx context.Context, rt controller.Runtime, Conditions: conditions, } - if resource.EqualStatus(res.Status[statusKeyArtistController], newStatus) { + if resource.EqualStatus(res.Status[statusKeyArtistController], newStatus, false) { return nil } diff --git a/internal/resource/equality.go b/internal/resource/equality.go index 100d4c7c95e2..c7c880cddc9e 100644 --- a/internal/resource/equality.go +++ b/internal/resource/equality.go @@ -52,7 +52,10 @@ func EqualID(a, b *pbresource.ID) bool { } // EqualStatus compares two statuses for equality without reflection. -func EqualStatus(a, b *pbresource.Status) bool { +// +// Pass true for compareUpdatedAt to compare the UpdatedAt timestamps, which you +// generally *don't* want when dirty checking the status in a controller. +func EqualStatus(a, b *pbresource.Status, compareUpdatedAt bool) bool { if a == b { return true } @@ -65,6 +68,10 @@ func EqualStatus(a, b *pbresource.Status) bool { return false } + if compareUpdatedAt && !a.UpdatedAt.AsTime().Equal(b.UpdatedAt.AsTime()) { + return false + } + if len(a.Conditions) != len(b.Conditions) { return false } @@ -125,7 +132,7 @@ func EqualStatusMap(a, b map[string]*pbresource.Status) bool { if !ok { return false } - if !EqualStatus(av, bv) { + if !EqualStatus(av, bv, true) { return false } compared[k] = struct{}{} @@ -141,7 +148,7 @@ func EqualStatusMap(a, b map[string]*pbresource.Status) bool { return false } - if !EqualStatus(av, bv) { + if !EqualStatus(av, bv, true) { return false } } diff --git a/internal/resource/equality_test.go b/internal/resource/equality_test.go index 45af9ea1801b..4fb7cb666b3a 100644 --- a/internal/resource/equality_test.go +++ b/internal/resource/equality_test.go @@ -6,10 +6,12 @@ package resource_test import ( "fmt" "testing" + "time" "github.com/oklog/ulid/v2" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/hashicorp/consul/internal/resource" "github.com/hashicorp/consul/proto-public/pbresource" @@ -310,17 +312,17 @@ func TestEqualStatus(t *testing.T) { // Equal cases. t.Run("same pointer", func(t *testing.T) { - require.True(t, resource.EqualStatus(orig, orig)) + require.True(t, resource.EqualStatus(orig, orig, true)) }) t.Run("equal", func(t *testing.T) { - require.True(t, resource.EqualStatus(orig, clone(orig))) + require.True(t, resource.EqualStatus(orig, clone(orig), true)) }) // Not equal cases. t.Run("nil", func(t *testing.T) { - require.False(t, resource.EqualStatus(orig, nil)) - require.False(t, resource.EqualStatus(nil, orig)) + require.False(t, resource.EqualStatus(orig, nil, true)) + require.False(t, resource.EqualStatus(nil, orig, true)) }) testCases := map[string]func(*pbresource.Status){ @@ -366,10 +368,24 @@ func TestEqualStatus(t *testing.T) { a, b := clone(orig), clone(orig) modFn(b) - require.False(t, resource.EqualStatus(a, b)) - require.False(t, resource.EqualStatus(b, a)) + require.False(t, resource.EqualStatus(a, b, true)) + require.False(t, resource.EqualStatus(b, a, true)) }) } + + t.Run("compareUpdatedAt = true", func(t *testing.T) { + a, b := clone(orig), clone(orig) + b.UpdatedAt = timestamppb.New(b.UpdatedAt.AsTime().Add(1 * time.Minute)) + require.False(t, resource.EqualStatus(a, b, true)) + require.False(t, resource.EqualStatus(b, a, true)) + }) + + t.Run("compareUpdatedAt = false", func(t *testing.T) { + a, b := clone(orig), clone(orig) + b.UpdatedAt = timestamppb.New(b.UpdatedAt.AsTime().Add(1 * time.Minute)) + require.True(t, resource.EqualStatus(a, b, false)) + require.True(t, resource.EqualStatus(b, a, false)) + }) } func TestEqualStatusMap(t *testing.T) { @@ -615,7 +631,7 @@ func BenchmarkEqualStatus(b *testing.B) { b.Run("ours", func(b *testing.B) { for i := 0; i < b.N; i++ { - _ = resource.EqualStatus(statusA, statusB) + _ = resource.EqualStatus(statusA, statusB, true) } }) From 5030101cdb0611160b38e633f4eecf64cd205966 Mon Sep 17 00:00:00 2001 From: Dan Upton Date: Wed, 10 May 2023 10:38:48 +0100 Subject: [PATCH 12/13] resource: add missing validation to the `List` and `WatchList` endpoints (#17213) --- agent/grpc-external/services/resource/list.go | 17 +++++++++++ .../services/resource/list_test.go | 25 +++++++++++++++++ .../grpc-external/services/resource/watch.go | 17 +++++++++++ .../services/resource/watch_test.go | 28 +++++++++++++++++++ 4 files changed, 87 insertions(+) diff --git a/agent/grpc-external/services/resource/list.go b/agent/grpc-external/services/resource/list.go index 65cc37a2691c..77269e74688f 100644 --- a/agent/grpc-external/services/resource/list.go +++ b/agent/grpc-external/services/resource/list.go @@ -15,6 +15,10 @@ import ( ) func (s *Server) List(ctx context.Context, req *pbresource.ListRequest) (*pbresource.ListResponse, error) { + if err := validateListRequest(req); err != nil { + return nil, err + } + // check type reg, err := s.resolveType(req.Type) if err != nil { @@ -65,3 +69,16 @@ func (s *Server) List(ctx context.Context, req *pbresource.ListRequest) (*pbreso } return &pbresource.ListResponse{Resources: result}, nil } + +func validateListRequest(req *pbresource.ListRequest) error { + var field string + switch { + case req.Type == nil: + field = "type" + case req.Tenancy == nil: + field = "tenancy" + default: + return nil + } + return status.Errorf(codes.InvalidArgument, "%s is required", field) +} diff --git a/agent/grpc-external/services/resource/list_test.go b/agent/grpc-external/services/resource/list_test.go index b476c82aef7b..7128d5e3167c 100644 --- a/agent/grpc-external/services/resource/list_test.go +++ b/agent/grpc-external/services/resource/list_test.go @@ -22,6 +22,31 @@ import ( "google.golang.org/grpc/status" ) +func TestList_InputValidation(t *testing.T) { + server := testServer(t) + client := testClient(t, server) + + demo.RegisterTypes(server.Registry) + + testCases := map[string]func(*pbresource.ListRequest){ + "no type": func(req *pbresource.ListRequest) { req.Type = nil }, + "no tenancy": func(req *pbresource.ListRequest) { req.Tenancy = nil }, + } + for desc, modFn := range testCases { + t.Run(desc, func(t *testing.T) { + req := &pbresource.ListRequest{ + Type: demo.TypeV2Album, + Tenancy: demo.TenancyDefault, + } + modFn(req) + + _, err := client.List(testContext(t), req) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument.String(), status.Code(err).String()) + }) + } +} + func TestList_TypeNotFound(t *testing.T) { server := testServer(t) client := testClient(t, server) diff --git a/agent/grpc-external/services/resource/watch.go b/agent/grpc-external/services/resource/watch.go index 77ffe19b0908..2fd943a6c946 100644 --- a/agent/grpc-external/services/resource/watch.go +++ b/agent/grpc-external/services/resource/watch.go @@ -13,6 +13,10 @@ import ( ) func (s *Server) WatchList(req *pbresource.WatchListRequest, stream pbresource.ResourceService_WatchListServer) error { + if err := validateWatchListRequest(req); err != nil { + return err + } + // check type exists reg, err := s.resolveType(req.Type) if err != nil { @@ -70,3 +74,16 @@ func (s *Server) WatchList(req *pbresource.WatchListRequest, stream pbresource.R } } } + +func validateWatchListRequest(req *pbresource.WatchListRequest) error { + var field string + switch { + case req.Type == nil: + field = "type" + case req.Tenancy == nil: + field = "tenancy" + default: + return nil + } + return status.Errorf(codes.InvalidArgument, "%s is required", field) +} diff --git a/agent/grpc-external/services/resource/watch_test.go b/agent/grpc-external/services/resource/watch_test.go index b62dc8a4079f..687fe0d0679f 100644 --- a/agent/grpc-external/services/resource/watch_test.go +++ b/agent/grpc-external/services/resource/watch_test.go @@ -22,6 +22,34 @@ import ( "google.golang.org/grpc/status" ) +func TestWatchList_InputValidation(t *testing.T) { + server := testServer(t) + client := testClient(t, server) + + demo.RegisterTypes(server.Registry) + + testCases := map[string]func(*pbresource.WatchListRequest){ + "no type": func(req *pbresource.WatchListRequest) { req.Type = nil }, + "no tenancy": func(req *pbresource.WatchListRequest) { req.Tenancy = nil }, + } + for desc, modFn := range testCases { + t.Run(desc, func(t *testing.T) { + req := &pbresource.WatchListRequest{ + Type: demo.TypeV2Album, + Tenancy: demo.TenancyDefault, + } + modFn(req) + + stream, err := client.WatchList(testContext(t), req) + require.NoError(t, err) + + _, err = stream.Recv() + require.Error(t, err) + require.Equal(t, codes.InvalidArgument.String(), status.Code(err).String()) + }) + } +} + func TestWatchList_TypeNotFound(t *testing.T) { t.Parallel() From e9986e37745959eea29bd7fc1e1f087090dc5446 Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 10 May 2023 14:49:18 -0600 Subject: [PATCH 13/13] ci:upload test results to datadog (#17206) * WIP * ci:upload test results to datadog * fix use of envvar in expression * getting correct permission in reusable-unit.yml * getting correct permission in reusable-unit.yml * fixing DATADOG_API_KEY envvar expresssion * pass datadog-api-key * removing type from datadog-api-key --- .github/workflows/go-tests.yml | 42 ++++++- .github/workflows/reusable-unit-split.yml | 33 +++++ .github/workflows/reusable-unit.yml | 33 +++++ .github/workflows/test-integrations.yml | 146 +++++++++++++++++++++- 4 files changed, 247 insertions(+), 7 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index d15b874ef671..787f92560ec1 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -21,7 +21,6 @@ permissions: env: TEST_RESULTS: /tmp/test-results - GOTESTSUM_VERSION: 1.8.2 jobs: setup: @@ -215,6 +214,7 @@ jobs: # secrets: # elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} # consul-license: ${{secrets.CONSUL_LICENSE}} + # datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-oss: needs: @@ -227,9 +227,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-enterprise: if: ${{ endsWith(github.repository, '-enterprise') }} @@ -243,9 +247,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-race: needs: @@ -259,9 +267,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-32bit: needs: @@ -275,9 +287,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-envoyextensions: needs: @@ -289,9 +305,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-troubleshoot: needs: @@ -303,9 +323,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-api-1-19: needs: @@ -317,9 +341,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-api-1-20: needs: @@ -331,9 +359,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-sdk-1-19: needs: @@ -345,9 +377,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-sdk-1-20: needs: @@ -359,9 +395,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" noop: runs-on: ubuntu-latest diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml index 0131582b0bef..f8088f4a8040 100644 --- a/.github/workflows/reusable-unit-split.yml +++ b/.github/workflows/reusable-unit-split.yml @@ -42,6 +42,8 @@ on: required: true consul-license: required: true + datadog-api-key: + required: true env: TEST_RESULTS: /tmp/test-results GOTESTSUM_VERSION: 1.8.2 @@ -49,6 +51,7 @@ env: TOTAL_RUNNERS: ${{inputs.runner-count}} CONSUL_LICENSE: ${{secrets.consul-license}} GOTAGS: ${{ inputs.go-tags}} + DATADOG_API_KEY: ${{secrets.datadog-api-key}} jobs: set-test-package-matrix: @@ -128,6 +131,36 @@ jobs: -tags="${{env.GOTAGS}}" -p 2 \ ${GO_TEST_FLAGS-} \ -cover -coverprofile=coverage.txt + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 with: name: test-results diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml index fde340bd5edc..5fd47339e12f 100644 --- a/.github/workflows/reusable-unit.yml +++ b/.github/workflows/reusable-unit.yml @@ -38,12 +38,15 @@ on: required: true consul-license: required: true + datadog-api-key: + required: true env: TEST_RESULTS: /tmp/test-results GOTESTSUM_VERSION: 1.8.2 GOARCH: ${{inputs.go-arch}} CONSUL_LICENSE: ${{secrets.consul-license}} GOTAGS: ${{ inputs.go-tags}} + DATADOG_API_KEY: ${{secrets.datadog-api-key}} jobs: go-test: @@ -96,6 +99,36 @@ jobs: -tags="${{env.GOTAGS}}" \ ${GO_TEST_FLAGS-} \ -cover -coverprofile=coverage.txt + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 with: name: test-results diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 33d1ccc9e207..460749fc971a 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -54,6 +54,9 @@ jobs: needs: - setup - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: matrix: nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16'] @@ -92,12 +95,45 @@ jobs: --packages="./command/agent/consul" \ --junitfile $TEST_RESULTS_DIR/results.xml -- \ -run TestConsul + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml vault-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} needs: - setup - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: matrix: vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"] @@ -139,6 +175,48 @@ jobs: --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \ -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" + + - name: upload leader coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" + + - name: upload agent coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" + generate-envoy-job-matrices: needs: [setup] runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} @@ -181,6 +259,9 @@ jobs: - setup - generate-envoy-job-matrices - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: fail-fast: false matrix: @@ -232,10 +313,35 @@ jobs: --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" - - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 with: - name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} - path: ${{ env.TEST_RESULTS_DIR }} + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml generate-compatibility-job-matrices: needs: [setup] @@ -274,6 +380,9 @@ jobs: - setup - dev-build - generate-compatibility-job-matrices + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: fail-fast: false matrix: @@ -335,10 +444,35 @@ jobs: # tput complains if this isn't set to something. TERM: ansi - - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 with: - name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} - path: ${{ env.TEST_RESULTS_DIR }} + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml generate-upgrade-job-matrices: needs: [setup]