[exporter/mezmoexporter] Remove hardcoded "otel" hostname

exporter/mezmoexporter/README.md

@@ -1,11 +1,14 @@
 # Mezmo Exporter
 
-This exporter supports sending OpenTelemetry log data to [LogDNA (Mezmo)](https://logdna.com).
+This exporter supports sending OpenTelemetry log data to
+[Mezmo](https://mezmo.com).
 
 # Configuration options:
 
-- `ingest_url` (optional): Specifies the URL to send ingested logs to.  If not specified, will default to `https://logs.logdna.com/logs/ingest`.
-- `ingest_key` (required): Ingestion key used to send log data to LogDNA.  See [Ingestion Keys](https://docs.logdna.com/docs/ingestion-key) for more details.
+- `ingest_url` (optional): Specifies the URL to send ingested logs to.  If not 
+specified, will default to `https://logs.mezmo.com/otel/ingest/rest`.
+- `ingest_key` (required): Ingestion key used to send log data to Mezmo.  See
+[Ingestion Keys](https://docs.mezmo.com/docs/ingestion-key) for more details.
 
 # Example:
 ## Simple Log Data
@@ -17,14 +20,23 @@ receivers:
       grpc:
         endpoint: ":4317"
 
+processors:
+  resourcedetection:
+    detectors:
+      - system
+    system:
+      hostname_sources:
+        - os
+
 exporters:
   mezmo:
-    ingest_url: "https://logs.logdna.com/logs/ingest"
+    ingest_url: "https://logs.mezmo.com/otel/ingest/rest"
     ingest_key: "00000000000000000000000000000000"
 
 service:
   pipelines:
     logs:
       receivers: [ otlp ]
+      processors: [ resourcedetection ]
       exporters: [ mezmo ]
 ```

exporter/mezmoexporter/config.go

@@ -17,6 +17,7 @@ package mezmoexporter // import "github.com/open-telemetry/opentelemetry-collect
 import (
 	"fmt"
 	"net/url"
+	"strings"
 	"time"
 
 	"go.opentelemetry.io/collector/config"
@@ -29,9 +30,9 @@ const (
 	defaultTimeout time.Duration = 5 * time.Second
 
 	// defaultIngestURL
-	defaultIngestURL = "https://logs.logdna.com/log/ingest"
+	defaultIngestURL = "https://logs.mezmo.com/otel/ingest/rest"
 
-	// See https://docs.logdna.com/docs/ingestion#service-limits for details
+	// See https://docs.mezmo.com/docs/Mezmo-ingestion-service-limits for details
 
 	// Maximum payload in bytes that can be POST'd to the REST endpoint
 	maxBodySize     = 10 * 1024 * 1024
@@ -68,11 +69,15 @@ func (c *Config) Validate() error {
 
 	parsed, err = url.Parse(c.IngestURL)
 	if c.IngestURL == "" || err != nil {
-		return fmt.Errorf("\"ingest_url\" must be a valid URL")
+		return fmt.Errorf(`"ingest_url" must be a valid URL`)
+	}
+
+	if !strings.HasSuffix(c.IngestURL, "/otel/ingest/rest") {
+		return fmt.Errorf(`"ingest_url" must end with "/otel/ingest/rest"`)
 	}
 
 	if parsed.Host == "" {
-		return fmt.Errorf("\"ingest_url\" must contain a valid host")
+		return fmt.Errorf(`"ingest_url" must contain a valid host`)
 	}
 
 	return nil

exporter/mezmoexporter/config_test.go

@@ -80,7 +80,7 @@ func TestLoadAllSettingsConfig(t *testing.T) {
 			NumConsumers: 7,
 			QueueSize:    17,
 		},
-		IngestURL: "https://alternate.logdna.com/log/ingest",
+		IngestURL: "https://alternate.mezmo.com/otel/ingest/rest",
 		IngestKey: "1234509876",
 	}
 	assert.Equal(t, &expectedCfg, e)

exporter/mezmoexporter/example/otel-collector-config.yaml

@@ -6,7 +6,7 @@ processors:
 
 exporters:
   mezmo:
-    ingest_url: "https://logs.logdna.com/log/ingest"
+    ingest_url: "https://logs.mezmo.com/otel/ingest/rest"
     ingest_key: "00000000000000000000000000000000"
 
 service:

exporter/mezmoexporter/exporter.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/pdata/pcommon"
@@ -86,17 +87,32 @@ func (m *mezmoExporter) logDataToMezmo(ld plog.Logs) error {
 	// Convert the log resources to mezmo lines...
 	resourceLogs := ld.ResourceLogs()
 	for i := 0; i < resourceLogs.Len(); i++ {
-		ills := resourceLogs.At(i).ScopeLogs()
-		for j := 0; j < ills.Len(); j++ {
-			logs := ills.At(j).LogRecords()
+		resource := resourceLogs.At(i).Resource()
+		resourceHostName, hasResourceHostName := resource.Attributes().Get("host.name")
+		scopeLogs := resourceLogs.At(i).ScopeLogs()
+
+		for j := 0; j < scopeLogs.Len(); j++ {
+			logs := scopeLogs.At(j).LogRecords()
 
 			for k := 0; k < logs.Len(); k++ {
 				log := logs.At(k)
 
 				// Convert Attributes to meta fields being mindful of the maxMetaDataSize restriction
 				attrs := map[string]string{}
-				attrs["trace.id"] = log.TraceID().HexString()
-				attrs["span.id"] = log.SpanID().HexString()
+				if hasResourceHostName {
+					attrs["hostname"] = resourceHostName.AsString()
+				}
+
+				traceID := log.TraceID().HexString()
+				if traceID != "" {
+					attrs["trace.id"] = traceID
+				}
+
+				spanID := log.SpanID().HexString()
+				if spanID != "" {
+					attrs["span.id"] = spanID
+				}
+
 				log.Attributes().Range(func(k string, v pcommon.Value) bool {
 					attrs[k] = truncateString(v.StringVal(), maxMetaDataSize)
 					return true
@@ -105,11 +121,21 @@ func (m *mezmoExporter) logDataToMezmo(ld plog.Logs) error {
 				s, _ := log.Attributes().Get("appname")
 				app := s.StringVal()
 
+				tstamp := log.Timestamp().AsTime().UTC().UnixMilli()
+				if tstamp == 0 {
+					tstamp = time.Now().UTC().UnixMilli()
+				}
+
+				logLevel := truncateString(log.SeverityText(), maxLogLevelLen)
+				if logLevel == "" {
+					logLevel = "info"
+				}
+
 				line := MezmoLogLine{
-					Timestamp: log.Timestamp().AsTime().UTC().UnixMilli(),
+					Timestamp: tstamp,
 					Line:      truncateString(log.Body().StringVal(), maxMessageSize),
 					App:       truncateString(app, maxAppnameLen),
-					Level:     truncateString(log.SeverityText(), maxLogLevelLen),
+					Level:     logLevel,
 					Meta:      attrs,
 				}
 				lines = append(lines, line)
@@ -154,12 +180,7 @@ func (m *mezmoExporter) logDataToMezmo(ld plog.Logs) error {
 }
 
 func (m *mezmoExporter) sendLinesToMezmo(post string) (errs error) {
-	// TODO When the Mezmo backend requirement to have a `hostname` value in the URI is removed, this hostname will no longer be needed.
-	var hostname = "otel"
-
-	url := fmt.Sprintf("%s?hostname=%s", m.config.IngestURL, hostname)
-
-	req, _ := http.NewRequest("POST", url, bytes.NewBuffer([]byte(post)))
+	req, _ := http.NewRequest("POST", m.config.IngestURL, bytes.NewBuffer([]byte(post)))
 	req.Header.Add("Accept", "application/json")
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("User-Agent", m.userAgentString)

exporter/mezmoexporter/exporter_test.go

@@ -62,6 +62,21 @@ func createSimpleLogData(numberOfLogs int) plog.Logs {
 	return logs
 }
 
+func createMinimalAttributesLogData(numberOfLogs int) plog.Logs {
+	logs := plog.NewLogs()
+	logs.ResourceLogs().AppendEmpty()
+	rl := logs.ResourceLogs().AppendEmpty()
+	rl.ScopeLogs().AppendEmpty()
+	sl := rl.ScopeLogs().AppendEmpty()
+
+	for i := 0; i < numberOfLogs; i++ {
+		logRecord := sl.LogRecords().AppendEmpty()
+		logRecord.Body().SetStringVal("minimal attribute log")
+	}
+
+	return logs
+}
+
 // Creates a logs set that exceeds the maximum message side we can send in one HTTP POST
 func createMaxLogData() plog.Logs {
 	logs := plog.NewLogs()
@@ -199,6 +214,38 @@ func TestLogsExporter(t *testing.T) {
 	})
 }
 
+func TestAddsRequiredAttributes(t *testing.T) {
+	httpServerParams := testServerParams{
+		t: t,
+		assertionsCallback: func(req *http.Request, body MezmoLogBody) (int, string) {
+			assert.Equal(t, "application/json", req.Header.Get("Content-Type"))
+			assert.Equal(t, "mezmo-otel-exporter/"+buildInfo.Version, req.Header.Get("User-Agent"))
+
+			lines := body.Lines
+			for _, line := range lines {
+				assert.True(t, line.Timestamp > 0)
+				assert.Equal(t, line.Level, "info")
+				assert.Equal(t, line.App, "")
+				assert.Equal(t, line.Line, "minimal attribute log")
+			}
+
+			return http.StatusOK, ""
+		},
+	}
+	server := createHTTPServer(&httpServerParams)
+	defer server.instance.Close()
+
+	log, _ := createLogger()
+	config := &Config{
+		IngestURL: server.url,
+	}
+	exporter := createExporter(t, config, log)
+
+	logs := createMinimalAttributesLogData(4)
+	err := exporter.pushLogData(context.Background(), logs)
+	require.NoError(t, err)
+}
+
 func Test404IngestError(t *testing.T) {
 	log, logObserver := createLogger()
 

exporter/mezmoexporter/factory_test.go

@@ -20,7 +20,6 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component/componenttest"
 	"go.opentelemetry.io/collector/config"
 	"go.opentelemetry.io/collector/config/confighttp"
@@ -52,11 +51,21 @@ func TestCreateDefaultConfig(t *testing.T) {
 	assert.NoError(t, configtest.CheckConfigStruct(cfg))
 }
 
-func TestCreateLogsExporter(t *testing.T) {
+func TestIngestUrlMustConform(t *testing.T) {
 	cfg := createDefaultConfig().(*Config)
 	cfg.IngestURL = "https://example.com:8088/services/collector"
 	cfg.IngestKey = "1234-1234"
 
+	params := componenttest.NewNopExporterCreateSettings()
+	_, err := createLogsExporter(context.Background(), params, cfg)
+	assert.Error(t, err, `"ingest_url" must end with "/otel/ingest/rest"`)
+}
+
+func TestCreateLogsExporter(t *testing.T) {
+	cfg := createDefaultConfig().(*Config)
+	cfg.IngestURL = "https://example.com:8088/otel/ingest/rest"
+	cfg.IngestKey = "1234-1234"
+
 	params := componenttest.NewNopExporterCreateSettings()
 	_, err := createLogsExporter(context.Background(), params, cfg)
 	assert.NoError(t, err)
@@ -80,19 +89,18 @@ func TestCreateInstanceViaFactory(t *testing.T) {
 	factory := NewFactory()
 
 	cfg := factory.CreateDefaultConfig().(*Config)
-	cfg.IngestURL = "https://example.com:8088/services/collector"
+	cfg.IngestURL = "https://example.com:8088/otel/ingest/rest"
 	cfg.IngestKey = "1234-1234"
 	params := componenttest.NewNopExporterCreateSettings()
 	exp, err := factory.CreateLogsExporter(context.Background(), params, cfg)
 	assert.NoError(t, err)
 	assert.NotNil(t, exp)
+	assert.NoError(t, exp.Shutdown(context.Background()))
 
 	// Set values that don't have a valid default.
 	cfg.IngestURL = "https://example.com"
 	cfg.IngestKey = "testToken"
 	exp, err = factory.CreateLogsExporter(context.Background(), params, cfg)
-	assert.NoError(t, err)
-	require.NotNil(t, exp)
-
-	assert.NoError(t, exp.Shutdown(context.Background()))
+	assert.Error(t, err)
+	assert.Nil(t, exp)
 }

exporter/mezmoexporter/testdata/config.yaml

@@ -8,7 +8,7 @@ exporters:
   mezmo:
     ingest_key: "00000000000000000000000000000000"
   mezmo/allsettings:
-    ingest_url: "https://alternate.logdna.com/log/ingest"
+    ingest_url: "https://alternate.mezmo.com/otel/ingest/rest"
     ingest_key: "1234509876"
     retry_on_failure:
       enabled: false

unreleased/mezmoexporter-hostname-removal.yaml

@@ -0,0 +1,29 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/mezmoexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: | 
+  This change removes the hardcoded "otel" hostname that was embedded
+  in outgoing logs data.
+
+# One or more tracking issues related to the change
+issues: [13410]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  It is replaced by:
+
+    1. Sending to a new collector endpoint that does not require the
+    hostname parameter.
+
+    2. Recognizing the "host.name" resource attribute and using that
+    value to fill in log metadata recognized upstream.
+
+    This is a breaking change, and as such will generate a startup
+    error if the exporter is configured to send to an endpoint that
+    does not support this feature.