[7.x] Index page URL and Referer as ECS fields (#3857) (#3865)

* Index page URL and Referer as ECS fields (#3857)

* Update approval files (#3867)

_meta/fields.common.yml

@@ -96,6 +96,11 @@
             description: >
               The canonical headers of the monitored HTTP request.
 
+          - name: referrer
+            type: keyword
+            ignore_above: 1024
+            description: Referrer for this HTTP request.
+
        - name: response
          type: group
          fields:

beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json

@@ -266,6 +266,7 @@
                         ]
                     },
                     "method": "post",
+                    "referrer": "http://localhost:8000/test/e2e/",
                     "socket": {
                         "encrypted": true,
                         "remote_address": "12.53.12.1"

beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json

@@ -219,6 +219,7 @@
                         ]
                     },
                     "method": "post",
+                    "referrer": "http://localhost:8000/test/e2e/",
                     "socket": {
                         "encrypted": true,
                         "remote_address": "12.53.12.1"

docs/data/elasticsearch/generated/errors.json

@@ -310,6 +310,7 @@
                     ]
                 },
                 "method": "post",
+                "referrer": "http://localhost:8000/test/e2e/",
                 "socket": {
                     "encrypted": true,
                     "remote_address": "8.8.8.8"

docs/data/elasticsearch/generated/transactions.json

@@ -357,6 +357,7 @@
                     ]
                 },
                 "method": "post",
+                "referrer": "http://localhost:8000/test/e2e/",
                 "socket": {
                     "encrypted": true,
                     "remote_address": "8.8.8.8"

docs/fields.asciidoc

@@ -177,6 +177,15 @@ Object is not enabled.
 
 --
 
+*`http.request.referrer`*::
++
+--
+Referrer for this HTTP request.
+
+type: keyword
+
+--
+
 
 *`http.response.status_code`*::
 +

model/context.go

@@ -19,6 +19,8 @@ package model
 
 import (
 	"net/http"
+	"net/url"
+	"strconv"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 
@@ -28,7 +30,7 @@ import (
 // Context holds all information sent under key context
 type Context struct {
 	Http         *Http
-	Url          *Url
+	URL          *URL
 	Labels       *Labels
 	Page         *Page
 	Custom       *Custom
@@ -43,8 +45,8 @@ type Http struct {
 	Response *Resp
 }
 
-// Url describes request URL and its components
-type Url struct {
+// URL describes an URL and its components
+type URL struct {
 	Original *string
 	Scheme   *string
 	Full     *string
@@ -55,9 +57,59 @@ type Url struct {
 	Fragment *string
 }
 
-// Page consists of Url string and referer
+func ParseURL(original, hostname string) *URL {
+	original = truncate(original)
+	url, err := url.Parse(original)
+	if err != nil {
+		return &URL{Original: &original}
+	}
+	if url.Scheme == "" {
+		url.Scheme = "http"
+	}
+	if url.Host == "" {
+		url.Host = hostname
+	}
+	full := truncate(url.String())
+	out := &URL{
+		Original: &original,
+		Scheme:   &url.Scheme,
+		Full:     &full,
+	}
+	if path := truncate(url.Path); path != "" {
+		out.Path = &path
+	}
+	if query := truncate(url.RawQuery); query != "" {
+		out.Query = &query
+	}
+	if fragment := url.Fragment; fragment != "" {
+		out.Fragment = &fragment
+	}
+	if host := truncate(url.Hostname()); host != "" {
+		out.Domain = &host
+	}
+	if port := truncate(url.Port()); port != "" {
+		if intv, err := strconv.Atoi(port); err == nil {
+			out.Port = &intv
+		}
+	}
+	return out
+}
+
+// truncate returns s truncated at n runes, and the number of runes in the resulting string (<= n).
+func truncate(s string) string {
+	var j int
+	for i := range s {
+		if j == 1024 {
+			return s[:i]
+		}
+		j++
+	}
+	return s
+}
+
+// Page consists of URL and referer
 type Page struct {
-	Url     *string
+	URL     *URL
 	Referer *string
 }
 
@@ -102,7 +154,7 @@ type MinimalResp struct {
 }
 
 // Fields returns common.MapStr holding transformed data for attribute url.
-func (url *Url) Fields() common.MapStr {
+func (url *URL) Fields() common.MapStr {
 	if url == nil {
 		return nil
 	}
@@ -146,7 +198,10 @@ func (page *Page) Fields() common.MapStr {
 		return nil
 	}
 	var fields = common.MapStr{}
-	utility.Set(fields, "url", page.Url)
+	// Remove in 8.0
+	if page.URL != nil {
+		utility.Set(fields, "url", page.URL.Original)
+	}
 	utility.Set(fields, "referer", page.Referer)
 	return fields
 }

model/error.go

@@ -62,7 +62,7 @@ type Error struct {
 	Labels  *Labels
 	Page    *Page
 	HTTP    *Http
-	URL     *Url
+	URL     *URL
 	Custom  *Custom
 
 	Exception *Exception
@@ -117,7 +117,16 @@ func (e *Error) Transform(ctx context.Context, tctx *transform.Context) []beat.E
 	// merges with metadata labels, overrides conflicting keys
 	utility.DeepUpdate(fields, "labels", e.Labels.Fields())
 	utility.Set(fields, "http", e.HTTP.Fields())
-	utility.Set(fields, "url", e.URL.Fields())
+	urlFields := e.URL.Fields()
+	if urlFields != nil {
+		utility.Set(fields, "url", e.URL.Fields())
+	}
+	if e.Page != nil {
+		utility.DeepUpdate(fields, "http.request.referrer", e.Page.Referer)
+		if urlFields == nil {
+			utility.Set(fields, "url", e.Page.URL.Fields())
+		}
+	}
 	utility.Set(fields, "experimental", e.Experimental)
 
 	// sampled and type is nil if an error happens outside a transaction or an (old) agent is not sending sampled info

model/error_test.go

@@ -357,7 +357,7 @@ func TestEvents(t *testing.T) {
 				TransactionID:      trID,
 				TransactionSampled: &sampledTrue,
 				Labels:             &labels,
-				Page:               &Page{Url: &url, Referer: &referer},
+				Page:               &Page{URL: &URL{Original: &url}, Referer: &referer},
 				Custom:             &custom,
 			},
 
@@ -388,6 +388,12 @@ func TestEvents(t *testing.T) {
 					}},
 					"page": common.MapStr{"url": url, "referer": referer},
 				},
+				"http": common.MapStr{
+					"request": common.MapStr{
+						"referrer": referer,
+					},
+				},
+				"url":         common.MapStr{"original": url},
 				"processor":   common.MapStr{"event": "error", "name": "error"},
 				"transaction": common.MapStr{"id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", "sampled": true},
 				"timestamp":   common.MapStr{"us": timestampUs},
@@ -532,6 +538,62 @@ func TestCulprit(t *testing.T) {
 	}
 }
 
+func TestErrorTransformPage(t *testing.T) {
+	id := "123"
+	urlExample := "http://example.com/path"
+
+	tests := []struct {
+		Error  Error
+		Output common.MapStr
+		Msg    string
+	}{
+		{
+			Error: Error{
+				ID: &id,
+				Page: &Page{
+					URL:     ParseURL(urlExample, ""),
+					Referer: nil,
+				},
+			},
+			Output: common.MapStr{
+				"domain":   "example.com",
+				"full":     "http://example.com/path",
+				"original": "http://example.com/path",
+				"path":     "/path",
+				"scheme":   "http",
+			},
+			Msg: "With page URL",
+		},
+		{
+			Error: Error{
+				ID:        &id,
+				Timestamp: time.Now(),
+				URL:       ParseURL("https://localhost:8200/", ""),
+				Page: &Page{
+					URL:     ParseURL(urlExample, ""),
+					Referer: nil,
+				},
+			},
+			Output: common.MapStr{
+				"domain":   "localhost",
+				"full":     "https://localhost:8200/",
+				"original": "https://localhost:8200/",
+				"path":     "/",
+				"port":     8200,
+				"scheme":   "https",
+			},
+			Msg: "With Page URL and Request URL",
+		},
+	}
+
+	tctx := &transform.Context{}
+
+	for idx, test := range tests {
+		output := test.Error.Transform(context.Background(), tctx)
+		assert.Equal(t, test.Output, output[0].Fields["url"], fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg))
+	}
+}
+
 func TestEmptyGroupingKey(t *testing.T) {
 	emptyGroupingKey := hex.EncodeToString(md5.New().Sum(nil))
 	e := Error{}

model/modeldecoder/context.go

@@ -54,7 +54,7 @@ func decodeContext(input map[string]interface{}, cfg Config, meta *model.Metadat
 
 	ctx := model.Context{
 		Http:         http,
-		Url:          url,
+		URL:          url,
 		Page:         page,
 		Custom:       custom,
 		Message:      message,
@@ -87,7 +87,7 @@ func decodeContext(input map[string]interface{}, cfg Config, meta *model.Metadat
 	return &ctx, nil
 }
 
-func decodeURL(raw common.MapStr, err error) (*model.Url, error) {
+func decodeURL(raw common.MapStr, err error) (*model.URL, error) {
 	if err != nil {
 		return nil, err
 	}
@@ -99,7 +99,7 @@ func decodeURL(raw common.MapStr, err error) (*model.Url, error) {
 	}
 
 	inpURL := decoder.MapStr(req, "url")
-	url := model.Url{
+	url := model.URL{
 		Original: decoder.StringPtr(inpURL, "raw"),
 		Full:     decoder.StringPtr(inpURL, "full"),
 		Domain:   decoder.StringPtr(inpURL, "hostname"),
@@ -217,10 +217,13 @@ func decodePage(raw common.MapStr, hasShortFieldNames bool, err error) (*model.P
 		return nil, nil
 	}
 	decoder := utility.ManualDecoder{}
-	return &model.Page{
-		Url:     decoder.StringPtr(pageInput, fieldName("url")),
+	page := &model.Page{
 		Referer: decoder.StringPtr(pageInput, fieldName("referer")),
-	}, decoder.Err
+	}
+	if pageURL := decoder.StringPtr(pageInput, fieldName("url")); pageURL != nil {
+		page.URL = model.ParseURL(*pageURL, "")
+	}
+	return page, decoder.Err
 }
 
 func decodeCustom(raw common.MapStr, hasShortFieldNames bool, err error) (*model.Custom, error) {

model/modeldecoder/error.go

@@ -73,7 +73,7 @@ func decodeError(input Input, schema *jsonschema.Schema) (*m.Error, error) {
 		Labels:             ctx.Labels,
 		Page:               ctx.Page,
 		HTTP:               ctx.Http,
-		URL:                ctx.Url,
+		URL:                ctx.URL,
 		Custom:             ctx.Custom,
 		Experimental:       ctx.Experimental,
 		Timestamp:          decoder.TimeEpochMicro(raw, "timestamp"),

model/modeldecoder/error_test.go

@@ -47,12 +47,12 @@ func TestErrorEventDecode(t *testing.T) {
 	transactionType := "request"
 	labels := m.Labels{"ab": "c"}
 	ua := "go-1.1"
-	page := m.Page{Url: &pURL, Referer: &referer}
+	page := m.Page{URL: m.ParseURL(pURL, ""), Referer: &referer}
 	custom := m.Custom{"a": "b"}
 	request := m.Req{Method: "post", Socket: &m.Socket{}, Headers: http.Header{"User-Agent": []string{ua}}, Cookies: map[string]interface{}{"a": "b"}}
 	response := m.Resp{Finished: new(bool), MinimalResp: m.MinimalResp{Headers: http.Header{"Content-Type": []string{"text/html"}}}}
 	h := m.Http{Request: &request, Response: &response}
-	ctxURL := m.Url{Original: &origURL}
+	ctxURL := m.URL{Original: &origURL}
 	inputMetadata := m.Metadata{
 		Service: m.Service{Name: "foo"},
 	}

model/modeldecoder/test_approved_model/context_client_ip_from_forwarded_header.approved.json

@@ -26,7 +26,7 @@
     "Labels": null,
     "Message": null,
     "Page": null,
-    "Url": {
+    "URL": {
         "Domain": null,
         "Fragment": null,
         "Full": null,

model/modeldecoder/test_approved_model/context_client_ip_from_socket.approved.json

@@ -19,7 +19,7 @@
     "Labels": null,
     "Message": null,
     "Page": null,
-    "Url": {
+    "URL": {
         "Domain": null,
         "Fragment": null,
         "Full": null,

model/modeldecoder/test_approved_model/context_client_ip_from_socket_invalid_headers.approved.json

@@ -23,7 +23,7 @@
     "Labels": null,
     "Message": null,
     "Page": null,
-    "Url": {
+    "URL": {
         "Domain": null,
         "Fragment": null,
         "Full": null,

model/modeldecoder/test_approved_model/context_client_ip_header_case_insensitive.approved.json

@@ -26,7 +26,7 @@
     "Labels": null,
     "Message": null,
     "Page": null,
-    "Url": {
+    "URL": {
         "Domain": null,
         "Fragment": null,
         "Full": null,

model/modeldecoder/test_approved_model/context_empty.approved.json

@@ -5,5 +5,5 @@
     "Labels": null,
     "Message": null,
     "Page": null,
-    "Url": null
+    "URL": null
 }