diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 23f3e39fd0c..a874a48cef0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -119,6 +119,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220] - Fix issue 17734 to retry on rate-limit error in the Filebeat httpjson input. {issue}17734[17734] {pull}17735[17735] - Remove migrationVersion map 7.7.0 reference from Kibana dashboard file to fix backward compatibility issues. {pull}17425[17425] +- Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847] *Heartbeat* diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index c588491f6f7..e946499c49e 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -5,6 +5,7 @@ package cloudfoundry import ( + "encoding/binary" "fmt" "net/url" "strings" @@ -377,18 +378,14 @@ func newEventBase(env *events.Envelope) eventBase { func newEventHttpAccess(env *events.Envelope) *EventHttpAccess { msg := env.GetHttpStartStop() - appID := "" - if msg.ApplicationId != nil { - appID = msg.ApplicationId.String() - } return &EventHttpAccess{ eventAppBase: eventAppBase{ eventBase: newEventBase(env), - appGuid: appID, + appGuid: formatUUID(msg.ApplicationId), }, startTimestamp: time.Unix(0, *msg.StartTimestamp), stopTimestamp: time.Unix(0, *msg.StopTimestamp), - requestID: msg.RequestId.String(), + requestID: formatUUID(msg.RequestId), peerType: strings.ToLower(msg.PeerType.String()), method: msg.Method.String(), uri: *msg.Uri, @@ -525,3 +522,13 @@ func urlMap(uri string) common.MapStr { "domain": u.Hostname(), } } + +func formatUUID(uuid *events.UUID) string { + if uuid == nil { + return "" + } + var uuidBytes [16]byte + binary.LittleEndian.PutUint64(uuidBytes[:8], uuid.GetLow()) + binary.LittleEndian.PutUint64(uuidBytes[8:], uuid.GetHigh()) + return fmt.Sprintf("%x-%x-%x-%x-%x", uuidBytes[0:4], uuidBytes[4:6], uuidBytes[6:8], uuidBytes[8:10], uuidBytes[10:]) +} diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go new file mode 100644 index 00000000000..e4fe6f39caf --- /dev/null +++ b/x-pack/libbeat/common/cloudfoundry/events_test.go @@ -0,0 +1,402 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package cloudfoundry + +import ( + "testing" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + + "github.com/cloudfoundry/sonde-go/events" + "github.com/stretchr/testify/assert" +) + +func TestEventTypeHttpAccess(t *testing.T) { + eventType := events.Envelope_HttpStartStop + startTimestamp := int64(1587469726082) + stopTimestamp := int64(1587469875895) + peerType := events.PeerType_Client + method := events.Method_GET + uri := "https://uri.full-domain.com:8443/subpath" + remoteAddress := "remote_address" + userAgent := "user_agent" + statusCode := int32(200) + contentLength := int64(128) + appID := makeUUID() + instanceIdx := int32(1) + instanceID := "instance_id" + forwarded := []string{"forwarded"} + cfEvt := makeEnvelope(&eventType) + cfEvt.HttpStartStop = &events.HttpStartStop{ + StartTimestamp: &startTimestamp, + StopTimestamp: &stopTimestamp, + RequestId: makeUUID(), + PeerType: &peerType, + Method: &method, + Uri: &uri, + RemoteAddress: &remoteAddress, + UserAgent: &userAgent, + StatusCode: &statusCode, + ContentLength: &contentLength, + ApplicationId: appID, + InstanceIndex: &instanceIdx, + InstanceId: &instanceID, + Forwarded: forwarded, + } + evt := newEventHttpAccess(cfEvt) + + assert.Equal(t, EventTypeHttpAccess, evt.EventType()) + assert.Equal(t, "access", evt.String()) + assert.Equal(t, "origin", evt.Origin()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp()) + assert.Equal(t, "deployment", evt.Deployment()) + assert.Equal(t, "job", evt.Job()) + assert.Equal(t, "index", evt.Index()) + assert.Equal(t, "ip", evt.IP()) + assert.Equal(t, map[string]string{"tag": "value"}, evt.Tags()) + assert.Equal(t, "f47ac10b-58cc-4372-a567-0e02b2c3d479", evt.AppGuid()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.StartTimestamp()) + assert.Equal(t, time.Unix(0, 1587469875895), evt.StopTimestamp()) + assert.Equal(t, "f47ac10b-58cc-4372-a567-0e02b2c3d479", evt.RequestID()) + assert.Equal(t, "client", evt.PeerType()) + assert.Equal(t, "GET", evt.Method()) + assert.Equal(t, "https://uri.full-domain.com:8443/subpath", evt.URI()) + assert.Equal(t, "remote_address", evt.RemoteAddress()) + assert.Equal(t, "user_agent", evt.UserAgent()) + assert.Equal(t, int32(200), evt.StatusCode()) + assert.Equal(t, int64(128), evt.ContentLength()) + assert.Equal(t, int32(1), evt.InstanceIndex()) + assert.Equal(t, []string{"forwarded"}, evt.Forwarded()) + + assert.Equal(t, common.MapStr{ + "cloudfoundry": common.MapStr{ + "type": "access", + "access": common.MapStr{ + "timestamp": time.Unix(0, 1587469726082), + }, + "envelope": common.MapStr{ + "origin": "origin", + "deployment": "deployment", + "ip": "ip", + "job": "job", + "index": "index", + }, + "app": common.MapStr{ + "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + }, + }, + "http": common.MapStr{ + "response": common.MapStr{ + "status_code": int32(200), + "method": "GET", + "bytes": int64(128), + }, + }, + "user_agent": common.MapStr{ + "original": "user_agent", + }, + "url": common.MapStr{ + "original": "https://uri.full-domain.com:8443/subpath", + "scheme": "https", + "port": "8443", + "path": "/subpath", + "domain": "uri.full-domain.com", + }, + }, evt.ToFields()) +} + +func TestEventTypeLog(t *testing.T) { + eventType := events.Envelope_LogMessage + message := "log message" + messageType := events.LogMessage_OUT + timestamp := int64(1587469726082) + appID := "f47ac10b-58cc-4372-a567-0e02b2c3d479" + sourceType := "source_type" + sourceInstance := "source_instance" + cfEvt := makeEnvelope(&eventType) + cfEvt.LogMessage = &events.LogMessage{ + Message: []byte(message), + MessageType: &messageType, + Timestamp: ×tamp, + AppId: &appID, + SourceType: &sourceType, + SourceInstance: &sourceInstance, + } + evt := newEventLog(cfEvt) + + assert.Equal(t, EventTypeLog, evt.EventType()) + assert.Equal(t, "log", evt.String()) + assert.Equal(t, "origin", evt.Origin()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp()) + assert.Equal(t, "deployment", evt.Deployment()) + assert.Equal(t, "job", evt.Job()) + assert.Equal(t, "index", evt.Index()) + assert.Equal(t, "ip", evt.IP()) + assert.Equal(t, map[string]string{"tag": "value"}, evt.Tags()) + assert.Equal(t, "f47ac10b-58cc-4372-a567-0e02b2c3d479", evt.AppGuid()) + assert.Equal(t, "log message", evt.Message()) + assert.Equal(t, EventLogMessageTypeStdout, evt.MessageType()) + assert.Equal(t, "source_type", evt.SourceType()) + assert.Equal(t, "source_instance", evt.SourceID()) + + assert.Equal(t, common.MapStr{ + "cloudfoundry": common.MapStr{ + "type": "log", + "log": common.MapStr{ + "timestamp": time.Unix(0, 1587469726082), + "source": common.MapStr{ + "instance": evt.SourceID(), + "type": evt.SourceType(), + }, + }, + "envelope": common.MapStr{ + "origin": "origin", + "deployment": "deployment", + "ip": "ip", + "job": "job", + "index": "index", + }, + "app": common.MapStr{ + "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + }, + }, + "message": "log message", + "stream": "stdout", + }, evt.ToFields()) +} + +func TestEventCounter(t *testing.T) { + eventType := events.Envelope_CounterEvent + name := "name" + delta := uint64(10) + total := uint64(999) + cfEvt := makeEnvelope(&eventType) + cfEvt.CounterEvent = &events.CounterEvent{ + Name: &name, + Delta: &delta, + Total: &total, + } + evt := newEventCounter(cfEvt) + + assert.Equal(t, EventTypeCounter, evt.EventType()) + assert.Equal(t, "counter", evt.String()) + assert.Equal(t, "origin", evt.Origin()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp()) + assert.Equal(t, "deployment", evt.Deployment()) + assert.Equal(t, "job", evt.Job()) + assert.Equal(t, "index", evt.Index()) + assert.Equal(t, "ip", evt.IP()) + assert.Equal(t, map[string]string{"tag": "value"}, evt.Tags()) + assert.Equal(t, "name", evt.Name()) + assert.Equal(t, uint64(10), evt.Delta()) + assert.Equal(t, uint64(999), evt.Total()) + + assert.Equal(t, common.MapStr{ + "cloudfoundry": common.MapStr{ + "type": "counter", + "counter": common.MapStr{ + "timestamp": time.Unix(0, 1587469726082), + "name": "name", + "delta": uint64(10), + "total": uint64(999), + }, + "envelope": common.MapStr{ + "origin": "origin", + "deployment": "deployment", + "ip": "ip", + "job": "job", + "index": "index", + }, + }, + }, evt.ToFields()) +} + +func TestEventValueMetric(t *testing.T) { + eventType := events.Envelope_ValueMetric + name := "name" + value := 10.1 + unit := "unit" + cfEvt := makeEnvelope(&eventType) + cfEvt.ValueMetric = &events.ValueMetric{ + Name: &name, + Value: &value, + Unit: &unit, + } + evt := newEventValueMetric(cfEvt) + + assert.Equal(t, EventTypeValueMetric, evt.EventType()) + assert.Equal(t, "value", evt.String()) + assert.Equal(t, "origin", evt.Origin()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp()) + assert.Equal(t, "deployment", evt.Deployment()) + assert.Equal(t, "job", evt.Job()) + assert.Equal(t, "index", evt.Index()) + assert.Equal(t, "ip", evt.IP()) + assert.Equal(t, map[string]string{"tag": "value"}, evt.Tags()) + assert.Equal(t, "name", evt.Name()) + assert.Equal(t, 10.1, evt.Value()) + assert.Equal(t, "unit", evt.Unit()) + + assert.Equal(t, common.MapStr{ + "cloudfoundry": common.MapStr{ + "type": "value", + "value": common.MapStr{ + "timestamp": time.Unix(0, 1587469726082), + "name": "name", + "value": 10.1, + "unit": "unit", + }, + "envelope": common.MapStr{ + "origin": "origin", + "deployment": "deployment", + "ip": "ip", + "job": "job", + "index": "index", + }, + }, + }, evt.ToFields()) +} + +func TestEventContainerMetric(t *testing.T) { + eventType := events.Envelope_ContainerMetric + appID := "f47ac10b-58cc-4372-a567-0e02b2c3d479" + instanceIdx := int32(1) + cpuPercentage := 0.2 + memoryBytes := uint64(1024) + diskBytes := uint64(2048) + memoryBytesQuota := uint64(2048) + diskBytesQuota := uint64(4096) + cfEvt := makeEnvelope(&eventType) + cfEvt.ContainerMetric = &events.ContainerMetric{ + ApplicationId: &appID, + InstanceIndex: &instanceIdx, + CpuPercentage: &cpuPercentage, + MemoryBytes: &memoryBytes, + DiskBytes: &diskBytes, + MemoryBytesQuota: &memoryBytesQuota, + DiskBytesQuota: &diskBytesQuota, + } + evt := newEventContainerMetric(cfEvt) + + assert.Equal(t, EventTypeContainerMetric, evt.EventType()) + assert.Equal(t, "container", evt.String()) + assert.Equal(t, "origin", evt.Origin()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp()) + assert.Equal(t, "deployment", evt.Deployment()) + assert.Equal(t, "job", evt.Job()) + assert.Equal(t, "index", evt.Index()) + assert.Equal(t, "ip", evt.IP()) + assert.Equal(t, map[string]string{"tag": "value"}, evt.Tags()) + assert.Equal(t, "f47ac10b-58cc-4372-a567-0e02b2c3d479", evt.AppGuid()) + assert.Equal(t, int32(1), evt.InstanceIndex()) + assert.Equal(t, 0.2, evt.CPUPercentage()) + assert.Equal(t, uint64(1024), evt.MemoryBytes()) + assert.Equal(t, uint64(2048), evt.DiskBytes()) + assert.Equal(t, uint64(2048), evt.MemoryBytesQuota()) + assert.Equal(t, uint64(4096), evt.DiskBytesQuota()) + + assert.Equal(t, common.MapStr{ + "cloudfoundry": common.MapStr{ + "type": "container", + "container": common.MapStr{ + "timestamp": time.Unix(0, 1587469726082), + "instance_index": int32(1), + "cpu.pct": 0.2, + "memory.bytes": uint64(1024), + "memory.quota.bytes": uint64(2048), + "disk.bytes": uint64(2048), + "disk.quota.bytes": uint64(4096), + }, + "envelope": common.MapStr{ + "origin": "origin", + "deployment": "deployment", + "ip": "ip", + "job": "job", + "index": "index", + }, + "app": common.MapStr{ + "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + }, + }, + }, evt.ToFields()) +} + +func TestEventError(t *testing.T) { + eventType := events.Envelope_Error + source := "source" + code := int32(100) + message := "message" + cfEvt := makeEnvelope(&eventType) + cfEvt.Error = &events.Error{ + Source: &source, + Code: &code, + Message: &message, + } + evt := newEventError(cfEvt) + + assert.Equal(t, EventTypeError, evt.EventType()) + assert.Equal(t, "error", evt.String()) + assert.Equal(t, "origin", evt.Origin()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp()) + assert.Equal(t, "deployment", evt.Deployment()) + assert.Equal(t, "job", evt.Job()) + assert.Equal(t, "index", evt.Index()) + assert.Equal(t, "ip", evt.IP()) + assert.Equal(t, map[string]string{"tag": "value"}, evt.Tags()) + assert.Equal(t, "message", evt.Message()) + assert.Equal(t, int32(100), evt.Code()) + assert.Equal(t, "source", evt.Source()) + + assert.Equal(t, common.MapStr{ + "cloudfoundry": common.MapStr{ + "type": "error", + "error": common.MapStr{ + "timestamp": time.Unix(0, 1587469726082), + "source": "source", + }, + "envelope": common.MapStr{ + "origin": "origin", + "deployment": "deployment", + "ip": "ip", + "job": "job", + "index": "index", + }, + }, + "message": "message", + "code": int32(100), + }, evt.ToFields()) +} + +func makeEnvelope(eventType *events.Envelope_EventType) *events.Envelope { + timestamp := int64(1587469726082) + origin := "origin" + deployment := "deployment" + job := "job" + index := "index" + ip := "ip" + return &events.Envelope{ + Origin: &origin, + EventType: eventType, + Timestamp: ×tamp, + Deployment: &deployment, + Job: &job, + Index: &index, + Ip: &ip, + Tags: map[string]string{"tag": "value"}, + } +} + +func makeUUID() *events.UUID { + // UUID `f47ac10b-58cc-4372-a567-0e02b2c3d479` + low := uint64(0x7243cc580bc17af4) + high := uint64(0x79d4c3b2020e67a5) + return &events.UUID{ + Low: &low, + High: &high, + } +}