Merge pull request #6699 from TheThingsNetwork/feature/gzip-experimental

Re-add gzip compression of HTTP server responses

CHANGELOG.md

@@ -11,6 +11,9 @@ For details about compatibility between different releases, see the **Commitment
 
 ### Added
 
+- The `http.client.transport.compression` experimental flag. It controls whether the HTTP clients used by the stack support gzip and zstd decompression of server responses. It is enabled by default.
+- The `http.server.transport.compression` experimental flag. It controls whether the HTTP servers used by the stack support gzip compression of the server response. It is enabled by default.
+
 ### Changed
 
 - The Things Stack is now built with Go 1.21.

pkg/experimental/experimental_test.go

@@ -24,6 +24,8 @@ import (
 )
 
 func TestExperimentalFeatures(t *testing.T) {
+	t.Parallel()
+
 	a := assertions.New(t)
 
 	r := NewRegistry()
@@ -32,21 +34,21 @@ func TestExperimentalFeatures(t *testing.T) {
 
 	feature := DefineFeature("experimental.feature", false)
 	a.So(feature.GetValue(ctx), should.BeFalse)
-	a.So(AllFeatures(ctx), should.Resemble, map[string]bool{"experimental.feature": false})
+	a.So(AllFeatures(ctx)["experimental.feature"], should.BeFalse)
 	a.So(feature.GetValue(context.Background()), should.BeFalse)
-	a.So(AllFeatures(context.Background()), should.Resemble, map[string]bool{"experimental.feature": false})
+	a.So(AllFeatures(context.Background())["experimental.feature"], should.BeFalse)
 
 	r.EnableFeatures("experimental.feature")
 	a.So(feature.GetValue(ctx), should.BeTrue)
-	a.So(AllFeatures(ctx), should.Resemble, map[string]bool{"experimental.feature": true})
+	a.So(AllFeatures(ctx)["experimental.feature"], should.BeTrue)
 	a.So(feature.GetValue(context.Background()), should.BeFalse)
-	a.So(AllFeatures(context.Background()), should.Resemble, map[string]bool{"experimental.feature": false})
+	a.So(AllFeatures(context.Background())["experimental.feature"], should.BeFalse)
 
 	EnableFeatures("experimental.feature")
 	r.DisableFeatures("experimental.feature")
 
 	a.So(feature.GetValue(ctx), should.BeFalse)
-	a.So(AllFeatures(ctx), should.Resemble, map[string]bool{"experimental.feature": false})
+	a.So(AllFeatures(ctx)["experimental.feature"], should.BeFalse)
 	a.So(feature.GetValue(context.Background()), should.BeTrue)
-	a.So(AllFeatures(context.Background()), should.Resemble, map[string]bool{"experimental.feature": true})
+	a.So(AllFeatures(context.Background())["experimental.feature"], should.BeTrue)
 }

pkg/httpclient/http.go

@@ -23,12 +23,16 @@ import (
 	"time"
 
 	"github.com/gregjones/httpcache"
+	"github.com/klauspost/compress/gzhttp"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.thethings.network/lorawan-stack/v3/pkg/config/tlsconfig"
+	"go.thethings.network/lorawan-stack/v3/pkg/experimental"
 	"go.thethings.network/lorawan-stack/v3/pkg/telemetry/tracing"
 	"go.thethings.network/lorawan-stack/v3/pkg/version"
 )
 
+var transportCompressionFeatureFlag = experimental.DefineFeature("http.client.transport.compression", true)
+
 // defaultHTTPClientTimeout is the default timeout for the HTTP client.
 const defaultHTTPClientTimeout = 10 * time.Second
 
@@ -95,11 +99,14 @@ func (p *provider) HTTPClient(ctx context.Context, opts ...Option) (*http.Client
 	transport := http.DefaultTransport.(*http.Transport).Clone()
 	transport.TLSClientConfig = options.tlsConfig
 
-	otelTransport := otelhttp.NewTransport(transport,
+	var rt http.RoundTripper = transport
+	if transportCompressionFeatureFlag.GetValue(ctx) {
+		rt = gzhttp.Transport(rt)
+	}
+	rt = otelhttp.NewTransport(
+		rt,
 		otelhttp.WithTracerProvider(tracing.FromContext(ctx)),
 	)
-
-	rt := http.RoundTripper(otelTransport)
 	if options.cache {
 		rt = &httpcache.Transport{
 			Transport:           rt,

pkg/web/web.go

@@ -27,8 +27,10 @@ import (
 
 	"github.com/gorilla/csrf"
 	"github.com/gorilla/mux"
+	"github.com/klauspost/compress/gzhttp"
 	"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
 	"go.thethings.network/lorawan-stack/v3/pkg/errors"
+	"go.thethings.network/lorawan-stack/v3/pkg/experimental"
 	"go.thethings.network/lorawan-stack/v3/pkg/fillcontext"
 	"go.thethings.network/lorawan-stack/v3/pkg/log"
 	"go.thethings.network/lorawan-stack/v3/pkg/random"
@@ -39,6 +41,19 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+var responseCompressionFeatureFlag = experimental.DefineFeature("http.server.transport.compression", true)
+
+func compressionMiddleware(ctx context.Context) (func(http.Handler) http.Handler, error) {
+	if !responseCompressionFeatureFlag.GetValue(ctx) {
+		return func(next http.Handler) http.Handler { return next }, nil
+	}
+	m, err := gzhttp.NewWrapper()
+	if err != nil {
+		return nil, err
+	}
+	return func(h http.Handler) http.Handler { return m(h) }, nil
+}
+
 // Registerer allows components to register their services to the web server.
 type Registerer interface {
 	RegisterRoutes(s *Server)
@@ -171,14 +186,21 @@ func New(ctx context.Context, opts ...Option) (*Server, error) {
 	}
 
 	var proxyConfiguration webmiddleware.ProxyConfiguration
-	proxyConfiguration.ParseAndAddTrusted(options.trustedProxies...)
+	if err := proxyConfiguration.ParseAndAddTrusted(options.trustedProxies...); err != nil {
+		return nil, err
+	}
+	compressor, err := compressionMiddleware(ctx)
+	if err != nil {
+		return nil, err
+	}
 	root := mux.NewRouter()
 	root.NotFoundHandler = http.HandlerFunc(webhandlers.NotFound)
 	root.Use(
 		webhandlers.WithErrorHandlers(map[string]http.Handler{
 			"text/html": webhandlers.Template,
 		}),
 		mux.MiddlewareFunc(webmiddleware.Recover()),
+		compressor,
 		otelmux.Middleware("ttn-lw-stack", otelmux.WithTracerProvider(tracing.FromContext(ctx))),
 		mux.MiddlewareFunc(webmiddleware.FillContext(options.contextFillers...)),
 		mux.MiddlewareFunc(webmiddleware.Peer()),