Backport of proxy-lifecycle: add HTTP Server with endpoints for proxy lifecycle shutdown into release/1.0.x

.changelog/115.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+Add HTTP server with configurable port and endpoint path for initiating graceful shutdown.
+```

.github/workflows/consul-dataplane-checks.yaml

@@ -1,28 +1,60 @@
 name: consul-dataplane-checks
-
 on:
   push:
     branches:
     - main
+    - 'release/*.*.x'
   pull_request:
 
 jobs:
+  get-go-version:
+    name: "Determine Go toolchain version"
+    runs-on: ubuntu-latest
+    outputs:
+      go-version: ${{ steps.get-go-version.outputs.go-version }}
+    steps:
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      - name: Determine Go version
+        id: get-go-version
+        # We use .go-version as our source of truth for current Go
+        # version, because "goenv" can react to it automatically.
+        run: |
+          echo "Building with Go $(cat .go-version)"
+          echo "go-version=$(cat .go-version)" >> $GITHUB_OUTPUT
   unit-tests:
     name: unit-tests
+    needs:
+      - get-go-version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+        with:
+          go-version: ${{ needs.get-go-version.outputs.go-version }}
+      - run: go test ./... -p 1 # disable parallelism to avoid port conflicts from default metrics and lifecycle server configuration
+  integration-tests:
+    name: integration-tests
+    needs:
+      - get-go-version
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
-          go-version: '1.20'
-      - run: go test ./...
+          go-version: ${{ needs.get-go-version.outputs.go-version }}
+      - run: make docker
+      #  Currently the server version below is set to 1.15-dev: integration-tests/main_test.go
+      - run: echo "VERSION=$(make version)" >> $GITHUB_ENV
+      - run: cd integration-tests && go test -dataplane-image="consul-dataplane/release-default:${{ env.VERSION }}"
   golangci:
     name: lint
+    needs:
+      - get-go-version
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
-          go-version: '1.20'
-      - uses: actions/checkout@v3
+          go-version: ${{ needs.get-go-version.outputs.go-version }}
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # v3.4.0

cmd/consul-dataplane/main.go

@@ -59,11 +59,13 @@ var (
 	promScrapePath        string
 	promMergePort         int
 
-	adminBindAddr    string
-	adminBindPort    int
-	readyBindAddr    string
-	readyBindPort    int
-	envoyConcurrency int
+	adminBindAddr         string
+	adminBindPort         int
+	readyBindAddr         string
+	readyBindPort         int
+	envoyConcurrency      int
+	envoyDrainTimeSeconds int
+	envoyDrainStrategy    string
 
 	xdsBindAddr string
 	xdsBindPort int
@@ -128,6 +130,8 @@ func init() {
 	StringVar(&readyBindAddr, "envoy-ready-bind-address", "", "DP_ENVOY_READY_BIND_ADDRESS", "The address on which Envoy's readiness probe is available.")
 	IntVar(&readyBindPort, "envoy-ready-bind-port", 0, "DP_ENVOY_READY_BIND_PORT", "The port on which Envoy's readiness probe is available.")
 	IntVar(&envoyConcurrency, "envoy-concurrency", 2, "DP_ENVOY_CONCURRENCY", "The number of worker threads that Envoy uses.")
+	IntVar(&envoyDrainTimeSeconds, "envoy-drain-time-seconds", 30, "DP_ENVOY_DRAIN_TIME", "The time in seconds for which Envoy will drain connections.")
+	StringVar(&envoyDrainStrategy, "envoy-drain-strategy", "immediate", "DP_ENVOY_DRAIN_STRATEGY", "The behaviour of Envoy during the drain sequence. Determines whether all open connections should be encouraged to drain immediately or to increase the percentage gradually as the drain time elapses.")
 
 	StringVar(&xdsBindAddr, "xds-bind-addr", "127.0.0.1", "DP_XDS_BIND_ADDR", "The address on which the Envoy xDS server is available.")
 	IntVar(&xdsBindPort, "xds-bind-port", 0, "DP_XDS_BIND_PORT", "The port on which the Envoy xDS server is available.")
@@ -232,6 +236,8 @@ func main() {
 			ReadyBindAddress:              readyBindAddr,
 			ReadyBindPort:                 readyBindPort,
 			EnvoyConcurrency:              envoyConcurrency,
+			EnvoyDrainTimeSeconds:         envoyDrainTimeSeconds,
+			EnvoyDrainStrategy:            envoyDrainStrategy,
 			ShutdownDrainListenersEnabled: shutdownDrainListenersEnabled,
 			ShutdownGracePeriodSeconds:    shutdownGracePeriodSeconds,
 			GracefulShutdownPath:          gracefulShutdownPath,

pkg/consuldp/config.go

@@ -271,7 +271,20 @@ type EnvoyConfig struct {
 	ReadyBindPort int
 	// EnvoyConcurrency is the envoy concurrency https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-concurrency
 	EnvoyConcurrency int
-	// ShutdownDrainListenersEnabled configures whether to wait for all proxy listeners to drain before terminating the proxy container.
+	// EnvoyDrainTime is the time in seconds for which Envoy will drain connections
+	// during a hot restart, when listeners are modified or removed via LDS, or when
+	// initiated manually via a request to the Envoy admin API.
+	// The Envoy HTTP connection manager filter will add “Connection: close” to HTTP1
+	// requests, send HTTP2 GOAWAY, and terminate connections on request completion
+	// (after the delayed close period).
+	// https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-drain-time-s
+	EnvoyDrainTimeSeconds int
+	// EnvoyDrainStrategy is the behaviour of Envoy during the drain sequence.
+	// Determines whether all open connections should be encouraged to drain
+	// immediately or to increase the percentage gradually as the drain time elapses.
+	// https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-drain-strategy
+	EnvoyDrainStrategy string
+	// ShutdownDrainListenersEnabled configures whether to start draining proxy listeners before terminating the proxy container. Drain time defaults to the value of ShutdownGracePeriodSeconds, but may be set explicitly with EnvoyDrainTimeSeconds.
 	ShutdownDrainListenersEnabled bool
 	// ShutdownGracePeriodSeconds is the amount of time to wait after receiving a SIGTERM before terminating the proxy container.
 	ShutdownGracePeriodSeconds int

pkg/consuldp/consul_dataplane.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -28,8 +29,9 @@ type xdsServer struct {
 	exitedCh        chan struct{}
 }
 
-type httpGetter interface {
+type httpClient interface {
 	Get(string) (*http.Response, error)
+	Post(string, string, io.Reader) (*http.Response, error)
 }
 
 // ConsulDataplane represents the consul-dataplane process
@@ -41,6 +43,7 @@ type ConsulDataplane struct {
 	xdsServer       *xdsServer
 	aclToken        string
 	metricsConfig   *metricsConfig
+	lifecycleConfig *lifecycleConfig
 }
 
 // NewConsulDP creates a new instance of ConsulDataplane
@@ -206,6 +209,12 @@ func (cdp *ConsulDataplane) Run(ctx context.Context) error {
 		return err
 	}
 
+	cdp.lifecycleConfig = NewLifecycleConfig(cdp.cfg, proxy)
+	err = cdp.lifecycleConfig.startLifecycleManager(ctx)
+	if err != nil {
+		return err
+	}
+
 	doneCh := make(chan error)
 	go func() {
 		select {
@@ -214,12 +223,25 @@ func (cdp *ConsulDataplane) Run(ctx context.Context) error {
 		case <-proxy.Exited():
 			doneCh <- errors.New("envoy proxy exited unexpectedly")
 		case <-cdp.xdsServerExited():
-			if err := proxy.Stop(); err != nil {
-				cdp.logger.Error("failed to stop proxy", "error", err)
+			// Initiate graceful shutdown of Envoy, kill if error
+			if err := proxy.Quit(); err != nil {
+				cdp.logger.Error("failed to stop proxy, will attempt to kill", "error", err)
+				if err := proxy.Kill(); err != nil {
+					cdp.logger.Error("failed to kill proxy", "error", err)
+				}
 			}
 			doneCh <- errors.New("xDS server exited unexpectedly")
 		case <-cdp.metricsConfig.metricsServerExited():
 			doneCh <- errors.New("metrics server exited unexpectedly")
+		case <-cdp.lifecycleConfig.lifecycleServerExited():
+			// Initiate graceful shutdown of Envoy, kill if error
+			if err := proxy.Quit(); err != nil {
+				cdp.logger.Error("failed to stop proxy", "error", err)
+				if err := proxy.Kill(); err != nil {
+					cdp.logger.Error("failed to kill proxy", "error", err)
+				}
+			}
+			doneCh <- errors.New("proxy lifecycle management server exited unexpectedly")
 		}
 	}()
 	return <-doneCh
@@ -247,20 +269,33 @@ func (cdp *ConsulDataplane) startDNSProxy(ctx context.Context) error {
 }
 
 func (cdp *ConsulDataplane) envoyProxyConfig(cfg []byte) envoy.ProxyConfig {
-	setConcurrency := true
 	extraArgs := cdp.cfg.Envoy.ExtraArgs
-	// Users could set the concurrency as an extra args. Take that as priority for best ux
-	// experience.
-	for _, v := range extraArgs {
-		if v == "--concurrency" {
-			setConcurrency = false
-		}
+
+	envoyArgs := map[string]interface{}{
+		"--concurrency":    cdp.cfg.Envoy.EnvoyConcurrency,
+		"--drain-time-s":   cdp.cfg.Envoy.EnvoyDrainTimeSeconds,
+		"--drain-strategy": cdp.cfg.Envoy.EnvoyDrainStrategy,
 	}
-	if setConcurrency {
-		extraArgs = append(extraArgs, fmt.Sprintf("--concurrency %v", cdp.cfg.Envoy.EnvoyConcurrency))
+
+	// Users could set the Envoy concurrency, drain time, or drain strategy as
+	// extra args. Prioritize values set in that way over passthrough or defaults
+	// from consul-dataplane.
+	for envoyArg, cdpEnvoyValue := range envoyArgs {
+		for _, v := range extraArgs {
+			// If found in extraArgs, skip setting value from consul-dataplane Envoy
+			// config
+			if v == envoyArg {
+				break
+			}
+		}
+
+		// If not found, append value from consul-dataplane Envoy config to extraArgs
+		extraArgs = append(extraArgs, fmt.Sprintf("%s %v", envoyArg, cdpEnvoyValue))
 	}
 
 	return envoy.ProxyConfig{
+		AdminAddr:       cdp.cfg.Envoy.AdminBindAddress,
+		AdminBindPort:   cdp.cfg.Envoy.AdminBindPort,
 		Logger:          cdp.logger,
 		LogJSON:         cdp.cfg.Logging.LogJSON,
 		BootstrapConfig: cfg,