From 0d9fb40bdbeabba5767340b06bb537949ded86dd Mon Sep 17 00:00:00 2001 From: Adrian-Stefan Mares Date: Thu, 9 Nov 2023 14:48:05 +0100 Subject: [PATCH 1/3] all: Add HTTP client transport compression --- CHANGELOG.md | 2 ++ pkg/httpclient/http.go | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69b28e3b9a..2d8e47d865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ For details about compatibility between different releases, see the **Commitment ### Added +- The `http.client.transport.compression` experimental flag. It controls whether the HTTP clients used by the stack support gzip and zstd decompression of server responses. It is enabled by default. + ### Changed - The Things Stack is now built with Go 1.21. diff --git a/pkg/httpclient/http.go b/pkg/httpclient/http.go index 94d5ec6b8d..0e6fe3b7bf 100644 --- a/pkg/httpclient/http.go +++ b/pkg/httpclient/http.go @@ -23,12 +23,16 @@ import ( "time" "github.com/gregjones/httpcache" + "github.com/klauspost/compress/gzhttp" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.thethings.network/lorawan-stack/v3/pkg/config/tlsconfig" + "go.thethings.network/lorawan-stack/v3/pkg/experimental" "go.thethings.network/lorawan-stack/v3/pkg/telemetry/tracing" "go.thethings.network/lorawan-stack/v3/pkg/version" ) +var transportCompressionFeatureFlag = experimental.DefineFeature("http.client.transport.compression", true) + // defaultHTTPClientTimeout is the default timeout for the HTTP client. const defaultHTTPClientTimeout = 10 * time.Second @@ -95,11 +99,14 @@ func (p *provider) HTTPClient(ctx context.Context, opts ...Option) (*http.Client transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = options.tlsConfig - otelTransport := otelhttp.NewTransport(transport, + var rt http.RoundTripper = transport + if transportCompressionFeatureFlag.GetValue(ctx) { + rt = gzhttp.Transport(rt) + } + rt = otelhttp.NewTransport( + rt, otelhttp.WithTracerProvider(tracing.FromContext(ctx)), ) - - rt := http.RoundTripper(otelTransport) if options.cache { rt = &httpcache.Transport{ Transport: rt, From 849badfa7a95ff0b5f4450aad036d0fb91e8c71d Mon Sep 17 00:00:00 2001 From: Adrian-Stefan Mares Date: Thu, 9 Nov 2023 15:31:33 +0100 Subject: [PATCH 2/3] all: Add HTTP server transport compression --- CHANGELOG.md | 1 + pkg/web/web.go | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8e47d865..232827f5bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ For details about compatibility between different releases, see the **Commitment ### Added - The `http.client.transport.compression` experimental flag. It controls whether the HTTP clients used by the stack support gzip and zstd decompression of server responses. It is enabled by default. +- The `http.server.transport.compression` experimental flag. It controls whether the HTTP servers used by the stack support gzip compression of the server response. It is enabled by default. ### Changed diff --git a/pkg/web/web.go b/pkg/web/web.go index ad2725d268..a5c1f0c270 100644 --- a/pkg/web/web.go +++ b/pkg/web/web.go @@ -27,8 +27,10 @@ import ( "github.com/gorilla/csrf" "github.com/gorilla/mux" + "github.com/klauspost/compress/gzhttp" "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" "go.thethings.network/lorawan-stack/v3/pkg/errors" + "go.thethings.network/lorawan-stack/v3/pkg/experimental" "go.thethings.network/lorawan-stack/v3/pkg/fillcontext" "go.thethings.network/lorawan-stack/v3/pkg/log" "go.thethings.network/lorawan-stack/v3/pkg/random" @@ -39,6 +41,19 @@ import ( "gopkg.in/yaml.v2" ) +var responseCompressionFeatureFlag = experimental.DefineFeature("http.server.transport.compression", true) + +func compressionMiddleware(ctx context.Context) (func(http.Handler) http.Handler, error) { + if !responseCompressionFeatureFlag.GetValue(ctx) { + return func(next http.Handler) http.Handler { return next }, nil + } + m, err := gzhttp.NewWrapper() + if err != nil { + return nil, err + } + return func(h http.Handler) http.Handler { return m(h) }, nil +} + // Registerer allows components to register their services to the web server. type Registerer interface { RegisterRoutes(s *Server) @@ -171,7 +186,13 @@ func New(ctx context.Context, opts ...Option) (*Server, error) { } var proxyConfiguration webmiddleware.ProxyConfiguration - proxyConfiguration.ParseAndAddTrusted(options.trustedProxies...) + if err := proxyConfiguration.ParseAndAddTrusted(options.trustedProxies...); err != nil { + return nil, err + } + compressor, err := compressionMiddleware(ctx) + if err != nil { + return nil, err + } root := mux.NewRouter() root.NotFoundHandler = http.HandlerFunc(webhandlers.NotFound) root.Use( @@ -179,6 +200,7 @@ func New(ctx context.Context, opts ...Option) (*Server, error) { "text/html": webhandlers.Template, }), mux.MiddlewareFunc(webmiddleware.Recover()), + compressor, otelmux.Middleware("ttn-lw-stack", otelmux.WithTracerProvider(tracing.FromContext(ctx))), mux.MiddlewareFunc(webmiddleware.FillContext(options.contextFillers...)), mux.MiddlewareFunc(webmiddleware.Peer()), From a4b92493831f8668f79143d453b33babce9a1753 Mon Sep 17 00:00:00 2001 From: Adrian-Stefan Mares Date: Thu, 9 Nov 2023 19:45:59 +0100 Subject: [PATCH 3/3] all: Fix experimental definition tests --- pkg/experimental/experimental_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/experimental/experimental_test.go b/pkg/experimental/experimental_test.go index ed18f25be3..5af0a94ced 100644 --- a/pkg/experimental/experimental_test.go +++ b/pkg/experimental/experimental_test.go @@ -24,6 +24,8 @@ import ( ) func TestExperimentalFeatures(t *testing.T) { + t.Parallel() + a := assertions.New(t) r := NewRegistry() @@ -32,21 +34,21 @@ func TestExperimentalFeatures(t *testing.T) { feature := DefineFeature("experimental.feature", false) a.So(feature.GetValue(ctx), should.BeFalse) - a.So(AllFeatures(ctx), should.Resemble, map[string]bool{"experimental.feature": false}) + a.So(AllFeatures(ctx)["experimental.feature"], should.BeFalse) a.So(feature.GetValue(context.Background()), should.BeFalse) - a.So(AllFeatures(context.Background()), should.Resemble, map[string]bool{"experimental.feature": false}) + a.So(AllFeatures(context.Background())["experimental.feature"], should.BeFalse) r.EnableFeatures("experimental.feature") a.So(feature.GetValue(ctx), should.BeTrue) - a.So(AllFeatures(ctx), should.Resemble, map[string]bool{"experimental.feature": true}) + a.So(AllFeatures(ctx)["experimental.feature"], should.BeTrue) a.So(feature.GetValue(context.Background()), should.BeFalse) - a.So(AllFeatures(context.Background()), should.Resemble, map[string]bool{"experimental.feature": false}) + a.So(AllFeatures(context.Background())["experimental.feature"], should.BeFalse) EnableFeatures("experimental.feature") r.DisableFeatures("experimental.feature") a.So(feature.GetValue(ctx), should.BeFalse) - a.So(AllFeatures(ctx), should.Resemble, map[string]bool{"experimental.feature": false}) + a.So(AllFeatures(ctx)["experimental.feature"], should.BeFalse) a.So(feature.GetValue(context.Background()), should.BeTrue) - a.So(AllFeatures(context.Background()), should.Resemble, map[string]bool{"experimental.feature": true}) + a.So(AllFeatures(context.Background())["experimental.feature"], should.BeTrue) }