Index page URL and Referer as ECS fields (elastic#3857)

* Split Page URL into ECS parts and store it separately

* Store page referer separately as ECS field

* Update changelog

_meta/fields.common.yml

@@ -96,6 +96,11 @@
  description: >
  The canonical headers of the monitored HTTP request.
 
+ - name: referrer
+ type: keyword
+ ignore_above: 1024
+ description: Referrer for this HTTP request.
+
  - name: response
  type: group
  fields:

beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json

@@ -266,6 +266,7 @@
  ]
  },
  "method": "post",
+ "referrer": "http://localhost:8000/test/e2e/",
  "socket": {
  "encrypted": true,
  "remote_address": "12.53.12.1"

beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json

@@ -219,6 +219,7 @@
  ]
  },
  "method": "post",
+ "referrer": "http://localhost:8000/test/e2e/",
  "socket": {
  "encrypted": true,
  "remote_address": "12.53.12.1"

docs/fields.asciidoc

@@ -177,6 +177,15 @@ Object is not enabled.
 
 --
 
+*`http.request.referrer`*::
++
+--
+Referrer for this HTTP request.
+
+type: keyword
+
+--
+
 
 *`http.response.status_code`*::
 +

model/context.go

@@ -19,6 +19,8 @@ package model
 
 import (
  "net/http"
+ "net/url"
+ "strconv"
 
  "github.com/elastic/beats/v7/libbeat/common"
 
@@ -28,7 +30,7 @@ import (
 // Context holds all information sent under key context
 type Context struct {
  Http *Http
- Url *Url
+ URL *URL
  Labels *Labels
  Page *Page
  Custom *Custom
@@ -43,8 +45,8 @@ type Http struct {
  Response *Resp
 }
 
-// Url describes request URL and its components
-type Url struct {
+// URL describes an URL and its components
+type URL struct {
  Original *string
  Scheme *string
  Full *string
@@ -55,9 +57,59 @@ type Url struct {
  Fragment *string
 }
 
-// Page consists of Url string and referer
+func ParseURL(original, hostname string) *URL {
+ original = truncate(original)
+ url, err := url.Parse(original)
+ if err != nil {
+ return &URL{Original: &original}
+ }
+ if url.Scheme == "" {
+ url.Scheme = "http"
+ }
+ if url.Host == "" {
+ url.Host = hostname
+ }
+ full := truncate(url.String())
+ out := &URL{
+ Original: &original,
+ Scheme: &url.Scheme,
+ Full: &full,
+ }
+ if path := truncate(url.Path); path != "" {
+ out.Path = &path
+ }
+ if query := truncate(url.RawQuery); query != "" {
+ out.Query = &query
+ }
+ if fragment := url.Fragment; fragment != "" {
+ out.Fragment = &fragment
+ }
+ if host := truncate(url.Hostname()); host != "" {
+ out.Domain = &host
+ }
+ if port := truncate(url.Port()); port != "" {
+ if intv, err := strconv.Atoi(port); err == nil {
+ out.Port = &intv
+ }
+ }
+ return out
+}
+
+// truncate returns s truncated at n runes, and the number of runes in the resulting string (<= n).
+func truncate(s string) string {
+ var j int
+ for i := range s {
+ if j == 1024 {
+ return s[:i]
+ }
+ j++
+ }
+ return s
+}
+
+// Page consists of URL and referer
 type Page struct {
- Url *string
+ URL *URL
  Referer *string
 }
 
@@ -102,7 +154,7 @@ type MinimalResp struct {
 }
 
 // Fields returns common.MapStr holding transformed data for attribute url.
-func (url *Url) Fields() common.MapStr {
+func (url *URL) Fields() common.MapStr {
  if url == nil {
  return nil
  }
@@ -146,7 +198,10 @@ func (page *Page) Fields() common.MapStr {
  return nil
  }
  var fields = common.MapStr{}
- utility.Set(fields, "url", page.Url)
+ // Remove in 8.0
+ if page.URL != nil {
+ utility.Set(fields, "url", page.URL.Original)
+ }
  utility.Set(fields, "referer", page.Referer)
  return fields
 }

model/error.go

@@ -62,7 +62,7 @@ type Error struct {
  Labels *Labels
  Page *Page
  HTTP *Http
- URL *Url
+ URL *URL
  Custom *Custom
 
  Exception *Exception
@@ -117,7 +117,16 @@ func (e *Error) Transform(ctx context.Context, tctx *transform.Context) []beat.E
  // merges with metadata labels, overrides conflicting keys
  utility.DeepUpdate(fields, "labels", e.Labels.Fields())
  utility.Set(fields, "http", e.HTTP.Fields())
- utility.Set(fields, "url", e.URL.Fields())
+ urlFields := e.URL.Fields()
+ if urlFields != nil {
+ utility.Set(fields, "url", e.URL.Fields())
+ }
+ if e.Page != nil {
+ utility.DeepUpdate(fields, "http.request.referrer", e.Page.Referer)
+ if urlFields == nil {
+ utility.Set(fields, "url", e.Page.URL.Fields())
+ }
+ }
  utility.Set(fields, "experimental", e.Experimental)
 
  // sampled and type is nil if an error happens outside a transaction or an (old) agent is not sending sampled info

model/error_test.go

@@ -357,7 +357,7 @@ func TestEvents(t *testing.T) {
  TransactionID: trID,
  TransactionSampled: &sampledTrue,
  Labels: &labels,
- Page: &Page{Url: &url, Referer: &referer},
+ Page: &Page{URL: &URL{Original: &url}, Referer: &referer},
  Custom: &custom,
  },
 
@@ -388,6 +388,12 @@ func TestEvents(t *testing.T) {
  }},
  "page": common.MapStr{"url": url, "referer": referer},
  },
+ "http": common.MapStr{
+ "request": common.MapStr{
+ "referrer": referer,
+ },
+ },
+ "url": common.MapStr{"original": url},
  "processor": common.MapStr{"event": "error", "name": "error"},
  "transaction": common.MapStr{"id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", "sampled": true},
  "timestamp": common.MapStr{"us": timestampUs},
@@ -532,6 +538,62 @@ func TestCulprit(t *testing.T) {
  }
 }
 
+func TestErrorTransformPage(t *testing.T) {
+ id := "123"
+ urlExample := "http://example.com/path"
+
+ tests := []struct {
+ Error Error
+ Output common.MapStr
+ Msg string
+ }{
+ {
+ Error: Error{
+ ID: &id,
+ Page: &Page{
+ URL: ParseURL(urlExample, ""),
+ Referer: nil,
+ },
+ },
+ Output: common.MapStr{
+ "domain": "example.com",
+ "full": "http://example.com/path",
+ "original": "http://example.com/path",
+ "path": "/path",
+ "scheme": "http",
+ },
+ Msg: "With page URL",
+ },
+ {
+ Error: Error{
+ ID: &id,
+ Timestamp: time.Now(),
+ URL: ParseURL("https://localhost:8200/", ""),
+ Page: &Page{
+ URL: ParseURL(urlExample, ""),
+ Referer: nil,
+ },
+ },
+ Output: common.MapStr{
+ "domain": "localhost",
+ "full": "https://localhost:8200/",
+ "original": "https://localhost:8200/",
+ "path": "/",
+ "port": 8200,
+ "scheme": "https",
+ },
+ Msg: "With Page URL and Request URL",
+ },
+ }
+
+ tctx := &transform.Context{}
+
+ for idx, test := range tests {
+ output := test.Error.Transform(context.Background(), tctx)
+ assert.Equal(t, test.Output, output[0].Fields["url"], fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg))
+ }
+}
+
 func TestEmptyGroupingKey(t *testing.T) {
  emptyGroupingKey := hex.EncodeToString(md5.New().Sum(nil))
  e := Error{}

model/modeldecoder/context.go

@@ -54,7 +54,7 @@ func decodeContext(input map[string]interface{}, cfg Config, meta *model.Metadat
 
  ctx := model.Context{
  Http: http,
- Url: url,
+ URL: url,
  Page: page,
  Custom: custom,
  Message: message,
@@ -87,7 +87,7 @@ func decodeContext(input map[string]interface{}, cfg Config, meta *model.Metadat
  return &ctx, nil
 }
 
-func decodeURL(raw common.MapStr, err error) (*model.Url, error) {
+func decodeURL(raw common.MapStr, err error) (*model.URL, error) {
  if err != nil {
  return nil, err
  }
@@ -99,7 +99,7 @@ func decodeURL(raw common.MapStr, err error) (*model.Url, error) {
  }
 
  inpURL := decoder.MapStr(req, "url")
- url := model.Url{
+ url := model.URL{
  Original: decoder.StringPtr(inpURL, "raw"),
  Full: decoder.StringPtr(inpURL, "full"),
  Domain: decoder.StringPtr(inpURL, "hostname"),
@@ -217,10 +217,13 @@ func decodePage(raw common.MapStr, hasShortFieldNames bool, err error) (*model.P
  return nil, nil
  }
  decoder := utility.ManualDecoder{}
- return &model.Page{
- Url: decoder.StringPtr(pageInput, fieldName("url")),
+ page := &model.Page{
  Referer: decoder.StringPtr(pageInput, fieldName("referer")),
- }, decoder.Err
+ }
+ if pageURL := decoder.StringPtr(pageInput, fieldName("url")); pageURL != nil {
+ page.URL = model.ParseURL(*pageURL, "")
+ }
+ return page, decoder.Err
 }
 
 func decodeCustom(raw common.MapStr, hasShortFieldNames bool, err error) (*model.Custom, error) {

model/modeldecoder/error.go

@@ -73,7 +73,7 @@ func decodeError(input Input, schema *jsonschema.Schema) (*m.Error, error) {
  Labels: ctx.Labels,
  Page: ctx.Page,
  HTTP: ctx.Http,
- URL: ctx.Url,
+ URL: ctx.URL,
  Custom: ctx.Custom,
  Experimental: ctx.Experimental,
  Timestamp: decoder.TimeEpochMicro(raw, "timestamp"),

model/modeldecoder/error_test.go

@@ -47,12 +47,12 @@ func TestErrorEventDecode(t *testing.T) {
  transactionType := "request"
  labels := m.Labels{"ab": "c"}
  ua := "go-1.1"
- page := m.Page{Url: &pURL, Referer: &referer}
+ page := m.Page{URL: m.ParseURL(pURL, ""), Referer: &referer}
  custom := m.Custom{"a": "b"}
  request := m.Req{Method: "post", Socket: &m.Socket{}, Headers: http.Header{"User-Agent": []string{ua}}, Cookies: map[string]interface{}{"a": "b"}}
  response := m.Resp{Finished: new(bool), MinimalResp: m.MinimalResp{Headers: http.Header{"Content-Type": []string{"text/html"}}}}
  h := m.Http{Request: &request, Response: &response}
- ctxURL := m.Url{Original: &origURL}
+ ctxURL := m.URL{Original: &origURL}
  inputMetadata := m.Metadata{
  Service: m.Service{Name: "foo"},
  }

model/modeldecoder/test_approved_model/context_client_ip_from_forwarded_header.approved.json

@@ -26,7 +26,7 @@
  "Labels": null,
  "Message": null,
  "Page": null,
- "Url": {
+ "URL": {
  "Domain": null,
  "Fragment": null,
  "Full": null,

model/modeldecoder/test_approved_model/context_client_ip_from_socket.approved.json

@@ -19,7 +19,7 @@
  "Labels": null,
  "Message": null,
  "Page": null,
- "Url": {
+ "URL": {
  "Domain": null,
  "Fragment": null,
  "Full": null,

model/modeldecoder/test_approved_model/context_client_ip_from_socket_invalid_headers.approved.json

@@ -23,7 +23,7 @@
  "Labels": null,
  "Message": null,
  "Page": null,
- "Url": {
+ "URL": {
  "Domain": null,
  "Fragment": null,
  "Full": null,

model/modeldecoder/test_approved_model/context_client_ip_header_case_insensitive.approved.json

@@ -26,7 +26,7 @@
  "Labels": null,
  "Message": null,
  "Page": null,
- "Url": {
+ "URL": {
  "Domain": null,
  "Fragment": null,
  "Full": null,

model/modeldecoder/test_approved_model/context_empty.approved.json

@@ -5,5 +5,5 @@
  "Labels": null,
  "Message": null,
  "Page": null,
- "Url": null
+ "URL": null
 }

model/modeldecoder/test_approved_model/context_experimental is not true.approved.json

@@ -5,5 +5,5 @@
  "Labels": null,
  "Message": null,
  "Page": null,
- "Url": null
+ "URL": null
 }