proxy-lifecycle: add HTTP Server with endpoints for proxy lifecycle shutdown #115
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| ```release-note:feature | ||
| Add HTTP server with configurable port and endpoint path for initiating graceful shutdown. | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,195 @@ | ||
| // Copyright (c) HashiCorp, Inc. | ||
| // SPDX-License-Identifier: MPL-2.0 | ||
|
|
||
| package consuldp | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "strconv" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/hashicorp/go-hclog" | ||
|
|
||
| "github.com/hashicorp/consul-dataplane/pkg/envoy" | ||
| ) | ||
|
|
||
| const ( | ||
| // defaultLifecycleBindPort is the port which will serve the proxy lifecycle HTTP | ||
| // endpoints on the loopback interface. | ||
| defaultLifecycleBindPort = "20300" | ||
| cdpLifecycleBindAddr = "127.0.0.1" | ||
| cdpLifecycleUrl = "http://" + cdpLifecycleBindAddr | ||
|
|
||
| defaultLifecycleShutdownPath = "/graceful_shutdown" | ||
| ) | ||
|
|
||
| // lifecycleConfig handles all configuration related to managing the Envoy proxy | ||
| // lifecycle, including exposing management controls via an HTTP server. | ||
| type lifecycleConfig struct { | ||
| logger hclog.Logger | ||
|
|
||
| // consuldp proxy lifecycle management config | ||
| shutdownDrainListenersEnabled bool | ||
| shutdownGracePeriodSeconds int | ||
| gracefulPort int | ||
| gracefulShutdownPath string | ||
|
|
||
| // manager for controlling the Envoy proxy process | ||
| proxy envoy.ProxyManager | ||
|
|
||
| // consuldp proxy lifecycle management server | ||
| lifecycleServer *http.Server | ||
|
|
||
| // consuldp proxy lifecycle server control | ||
| errorExitCh chan struct{} | ||
| running bool | ||
| mu sync.Mutex | ||
| } | ||
|
|
||
| func NewLifecycleConfig(cfg *Config, proxy envoy.ProxyManager) *lifecycleConfig { | ||
| return &lifecycleConfig{ | ||
| shutdownDrainListenersEnabled: cfg.Envoy.ShutdownDrainListenersEnabled, | ||
| shutdownGracePeriodSeconds: cfg.Envoy.ShutdownGracePeriodSeconds, | ||
| gracefulPort: cfg.Envoy.GracefulPort, | ||
| gracefulShutdownPath: cfg.Envoy.GracefulShutdownPath, | ||
|
|
||
| proxy: proxy, | ||
|
|
||
| errorExitCh: make(chan struct{}, 1), | ||
mikemorris marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| mu: sync.Mutex{}, | ||
| } | ||
| } | ||
|
|
||
| func (m *lifecycleConfig) startLifecycleManager(ctx context.Context) error { | ||
| m.mu.Lock() | ||
| defer m.mu.Unlock() | ||
| if m.running { | ||
| return nil | ||
| } | ||
|
|
||
| m.logger = hclog.FromContext(ctx).Named("lifecycle") | ||
| m.running = true | ||
| go func() { | ||
| <-ctx.Done() | ||
| m.stopLifecycleServer() | ||
| }() | ||
|
|
||
| // Start the server which will expose HTTP endpoints for proxy lifecycle | ||
| // management control | ||
| mux := http.NewServeMux() | ||
|
|
||
| // Determine what HTTP endpoint paths to configure for the proxy lifecycle | ||
| // management server. These can be set as flags. | ||
| cdpLifecycleShutdownPath := defaultLifecycleShutdownPath | ||
| if m.gracefulShutdownPath != "" { | ||
| cdpLifecycleShutdownPath = m.gracefulShutdownPath | ||
| } | ||
|
|
||
| // Set config to allow introspection of default path for testing | ||
| m.gracefulShutdownPath = cdpLifecycleShutdownPath | ||
|
|
||
| m.logger.Info(fmt.Sprintf("setting graceful shutdown path: %s\n", cdpLifecycleShutdownPath)) | ||
| mux.HandleFunc(cdpLifecycleShutdownPath, m.gracefulShutdown) | ||
|
|
||
| // Determine what the proxy lifecycle management server bind port is. It can be | ||
| // set as a flag. | ||
| cdpLifecycleBindPort := defaultLifecycleBindPort | ||
| if m.gracefulPort != 0 { | ||
| cdpLifecycleBindPort = strconv.Itoa(m.gracefulPort) | ||
| } | ||
| m.lifecycleServer = &http.Server{ | ||
| Addr: fmt.Sprintf("%s:%s", cdpLifecycleBindAddr, cdpLifecycleBindPort), | ||
| Handler: mux, | ||
| } | ||
|
|
||
| // Start the proxy lifecycle management server | ||
| go m.startLifecycleServer() | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // startLifecycleServer starts the main proxy lifecycle management server that | ||
| // exposes HTTP endpoints for proxy lifecycle control. | ||
| func (m *lifecycleConfig) startLifecycleServer() { | ||
| m.logger.Info("starting proxy lifecycle management server", "address", m.lifecycleServer.Addr) | ||
| err := m.lifecycleServer.ListenAndServe() | ||
| if err != nil && err != http.ErrServerClosed { | ||
| m.logger.Error("failed to serve proxy lifecycle management requests", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| } | ||
|
|
||
| // stopLifecycleServer stops the consul dataplane proxy lifecycle server | ||
| func (m *lifecycleConfig) stopLifecycleServer() { | ||
| m.mu.Lock() | ||
| defer m.mu.Unlock() | ||
| m.running = false | ||
|
|
||
| if m.lifecycleServer != nil { | ||
| m.logger.Info("stopping the lifecycle management server") | ||
| err := m.lifecycleServer.Close() | ||
| if err != nil { | ||
| m.logger.Warn("error while closing lifecycle server", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // lifecycleServerExited is used to signal that the lifecycle server | ||
| // recieved a signal to initiate shutdown. | ||
| func (m *lifecycleConfig) lifecycleServerExited() <-chan struct{} { | ||
| return m.errorExitCh | ||
| } | ||
|
|
||
| // gracefulShutdown blocks until shutdownGracePeriodSeconds seconds have elapsed, and, if | ||
| // configured, will drain inbound connections to Envoy listeners during that time. | ||
| func (m *lifecycleConfig) gracefulShutdown(rw http.ResponseWriter, _ *http.Request) { | ||
pglass marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| m.logger.Info("initiating shutdown") | ||
|
|
||
| // Create a context that will signal a cancel at the specified duration. | ||
| // TODO: should this use lifecycleManager ctx instead of context.Background? | ||
| timeout := time.Duration(m.shutdownGracePeriodSeconds) * time.Second | ||
| ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
|
|
||
| m.logger.Info(fmt.Sprintf("waiting %d seconds before terminating dataplane proxy", m.shutdownGracePeriodSeconds)) | ||
|
|
||
| var wg sync.WaitGroup | ||
| wg.Add(1) | ||
|
|
||
| go func() { | ||
| defer wg.Done() | ||
|
|
||
| // If shutdownDrainListenersEnabled, initiatie graceful shutdown of Envoy. | ||
| // We want to start draining connections from inbound listeners if | ||
| // configured, but still allow outbound traffic until gracefulShutdownPeriod | ||
| // has elapsed to facilitate a graceful application shutdown. | ||
| if m.shutdownDrainListenersEnabled { | ||
| err := m.proxy.Drain() | ||
| if err != nil { | ||
| m.logger.Warn("error while draining Envoy listeners", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| } | ||
|
|
||
| // Block until context timeout has elapsed | ||
| <-ctx.Done() | ||
|
|
||
| // Finish graceful shutdown, quit Envoy proxy | ||
| m.logger.Info("shutdown grace period timeout reached") | ||
| err := m.proxy.Quit() | ||
| if err != nil { | ||
| m.logger.Warn("error while shutting down Envoy", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| }() | ||
|
|
||
| // Wait for context timeout to elapse | ||
| wg.Wait() | ||
pglass marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| // Return HTTP 200 Success | ||
| rw.WriteHeader(http.StatusOK) | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What kind of impact does this have on the runtime for these tests?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'd like to clean this up, but skipped for now in the interest of expediency. It didn't feel substantial enough to warrant the effort at this time, as the full suite still completes in under a minute.