Make gzip default for otlp gRPC/HTTP exporters (#4632)

Change OTLP gRPC exporter to use `gzip` compression by default. If this is merged, I can followup with the OTLP HTTP exporter to use `gzip` by default as well. **Link to tracking Issue:** #4587 **Testing:** - Added benchmarks comparing different compression algorithms. - Updated `configgrpc_test.go` to validate default behavior **Documentation:** Added details to `/config/configgrpc/README.md` with benchmark results and notes on compression.
open-telemetry · Feb 3, 2022 · 5470057 · 5470057
1 parent 9b2d254
commit 5470057
Show file tree

Hide file tree

Showing 9 changed files with 86 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,7 @@
   in 0.11.0 are no longer converted to the messages and fields that replaced the deprecated ones.
   Received deprecated messages and fields will be now ignored. In OTLP/JSON in the
   instrumentationLibraryLogs object the "logs" field is now named "logRecords" (#4724)
+- `otlphttp` and `otlp` exporters enable gzip compression by default (#4632)
 
 ## 💡 Enhancements 💡
 

diff --git a/config/configgrpc/README.md b/config/configgrpc/README.md
@@ -16,7 +16,7 @@ configuration. For more information, see [configtls
 README](../configtls/README.md).
 
 - [`balancer_name`](https://github.com/grpc/grpc-go/blob/master/examples/features/load_balancing/README.md)
-- `compression` Compression type to use among `gzip`, `snappy` and `zstd`
+- `compression` Compression type to use among `gzip`, `snappy`, `zstd`, and `none`.
 - `endpoint`: Valid value syntax available [here](https://github.com/grpc/grpc/blob/master/doc/naming.md)
 - [`tls`](../configtls/README.md)
 - `headers`: name/value pairs added to the request

diff --git a/exporter/otlpexporter/README.md b/exporter/otlpexporter/README.md
@@ -46,6 +46,15 @@ exporters:
       insecure: true
 ```
 
+By default `gzip` compression is enabled. See [compression comparison](../../config/configgrpc/README.md#compression-comparison) for details benchmark information. To disable, configure as follows:
+
+```yaml
+exporters:
+  otlp:
+    ...
+    compression: none
+```
+
 ## Advanced Configuration
 
 Several helper files are leveraged to provide additional capabilities automatically:

diff --git a/exporter/otlpexporter/factory.go b/exporter/otlpexporter/factory.go
@@ -19,6 +19,7 @@ import (
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/configcompression"
 	"go.opentelemetry.io/collector/config/configgrpc"
 	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
@@ -47,6 +48,8 @@ func createDefaultConfig() config.Exporter {
 		QueueSettings:    exporterhelper.DefaultQueueSettings(),
 		GRPCClientSettings: configgrpc.GRPCClientSettings{
 			Headers: map[string]string{},
+			// Default to gzip compression
+			Compression: configcompression.Gzip,
 			// We almost read 0 bytes, so no need to tune ReadBufferSize.
 			WriteBufferSize: 512 * 1024,
 		},

diff --git a/exporter/otlpexporter/factory_test.go b/exporter/otlpexporter/factory_test.go
@@ -43,6 +43,7 @@ func TestCreateDefaultConfig(t *testing.T) {
 	assert.Equal(t, ocfg.RetrySettings, exporterhelper.DefaultRetrySettings())
 	assert.Equal(t, ocfg.QueueSettings, exporterhelper.DefaultQueueSettings())
 	assert.Equal(t, ocfg.TimeoutSettings, exporterhelper.DefaultTimeoutSettings())
+	assert.Equal(t, ocfg.Compression, configcompression.Gzip)
 }
 
 func TestCreateMetricsExporter(t *testing.T) {
@@ -100,6 +101,16 @@ func TestCreateTracesExporter(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "NoneCompression",
+			config: Config{
+				ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+				GRPCClientSettings: configgrpc.GRPCClientSettings{
+					Endpoint:    endpoint,
+					Compression: "none",
+				},
+			},
+		},
 		{
 			name: "GzipCompression",
 			config: Config{

diff --git a/exporter/otlphttpexporter/README.md b/exporter/otlphttpexporter/README.md
@@ -42,5 +42,14 @@ exporters:
     endpoint: https://example.com:4318/v1/traces
 ```
 
+By default `gzip` compression is enabled. See [compression comparison](../../config/configgrpc/README.md#compression-comparison) for details benchmark information. To disable, configure as follows:
+
+```yaml
+exporters:
+  otlphttp:
+    ...
+    compression: none
+```
+
 The full list of settings exposed for this exporter are documented [here](./config.go)
 with detailed sample configurations [here](./testdata/config.yaml).
diff --git a/exporter/otlphttpexporter/factory.go b/exporter/otlphttpexporter/factory.go
@@ -22,6 +22,7 @@ import (
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/configcompression"
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
@@ -51,6 +52,8 @@ func createDefaultConfig() config.Exporter {
 			Endpoint: "",
 			Timeout:  30 * time.Second,
 			Headers:  map[string]string{},
+			// Default to gzip compression
+			Compression: configcompression.Gzip,
 			// We almost read 0 bytes, so no need to tune ReadBufferSize.
 			WriteBufferSize: 512 * 1024,
 		},

diff --git a/exporter/otlphttpexporter/factory_test.go b/exporter/otlphttpexporter/factory_test.go
@@ -25,6 +25,7 @@ import (
 
 	"go.opentelemetry.io/collector/component/componenttest"
 	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/configcompression"
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/configtest"
 	"go.opentelemetry.io/collector/config/configtls"
@@ -45,6 +46,7 @@ func TestCreateDefaultConfig(t *testing.T) {
 	assert.Equal(t, ocfg.RetrySettings.InitialInterval, 5*time.Second, "default retry InitialInterval")
 	assert.Equal(t, ocfg.RetrySettings.MaxInterval, 30*time.Second, "default retry MaxInterval")
 	assert.Equal(t, ocfg.QueueSettings.Enabled, true, "default sending queue is enabled")
+	assert.Equal(t, ocfg.Compression, configcompression.Gzip)
 }
 
 func TestCreateMetricsExporter(t *testing.T) {
@@ -132,6 +134,46 @@ func TestCreateTracesExporter(t *testing.T) {
 			mustFailOnCreate: false,
 			mustFailOnStart:  true,
 		},
+		{
+			name: "NoneCompression",
+			config: Config{
+				ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+				HTTPClientSettings: confighttp.HTTPClientSettings{
+					Endpoint:    endpoint,
+					Compression: "none",
+				},
+			},
+		},
+		{
+			name: "GzipCompression",
+			config: Config{
+				ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+				HTTPClientSettings: confighttp.HTTPClientSettings{
+					Endpoint:    endpoint,
+					Compression: configcompression.Gzip,
+				},
+			},
+		},
+		{
+			name: "SnappyCompression",
+			config: Config{
+				ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+				HTTPClientSettings: confighttp.HTTPClientSettings{
+					Endpoint:    endpoint,
+					Compression: configcompression.Snappy,
+				},
+			},
+		},
+		{
+			name: "ZstdCompression",
+			config: Config{
+				ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+				HTTPClientSettings: confighttp.HTTPClientSettings{
+					Endpoint:    endpoint,
+					Compression: configcompression.Zstd,
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

diff --git a/exporter/otlphttpexporter/otlp_test.go b/exporter/otlphttpexporter/otlp_test.go
@@ -15,6 +15,8 @@
 package otlphttpexporter
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/base64"
 	"encoding/hex"
@@ -243,7 +245,11 @@ func TestLogsRoundTrip(t *testing.T) {
 func TestIssue_4221(t *testing.T) {
 	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer func() { assert.NoError(t, r.Body.Close()) }()
-		data, err := ioutil.ReadAll(r.Body)
+		compressedData, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		gzipReader, err := gzip.NewReader(bytes.NewReader(compressedData))
+		require.NoError(t, err)
+		data, err := ioutil.ReadAll(gzipReader)
 		require.NoError(t, err)
 		base64Data := base64.StdEncoding.EncodeToString(data)
 		// Verify same base64 encoded string is received.